diff --git a/poetry.lock b/poetry.lock index 8ef88d85..0df78020 100644 --- a/poetry.lock +++ b/poetry.lock @@ -135,6 +135,39 @@ files = [ {file = "altgraph-0.17.4.tar.gz", hash = "sha256:1b5afbb98f6c4dcadb2e2ae6ab9fa994bbb8c1d75f4fa96d340f9437ae454406"}, ] +[[package]] +name = "annotated-types" +version = "0.6.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +files = [ + {file = "annotated_types-0.6.0-py3-none-any.whl", hash = "sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43"}, + {file = "annotated_types-0.6.0.tar.gz", hash = "sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d"}, +] + +[[package]] +name = "anyio" +version = "4.2.0" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +optional = false +python-versions = ">=3.8" +files = [ + {file = "anyio-4.2.0-py3-none-any.whl", hash = "sha256:745843b39e829e108e518c489b31dc757de7d2131d53fac32bd8df268227bfee"}, + {file = "anyio-4.2.0.tar.gz", hash = "sha256:e1875bb4b4e2de1669f4bc7869b6d3f54231cdced71605e6e64c9be77e3be50f"}, +] + +[package.dependencies] +exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} +idna = ">=2.8" +sniffio = ">=1.1" +typing-extensions = {version = ">=4.1", markers = "python_version < \"3.11\""} + +[package.extras] +doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] +trio = ["trio (>=0.23)"] + [[package]] name = "asn1crypto" version = "1.5.1" @@ -1304,6 +1337,17 @@ gitdb = ">=4.0.1,<5" [package.extras] test = ["black", "coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "sumtypes"] +[[package]] +name = "h11" +version = "0.14.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = false +python-versions = ">=3.7" +files = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] + [[package]] name = "hexbytes" version = "0.3.1" @@ -1323,13 +1367,13 @@ test = ["eth-utils (>=1.0.1,<3)", "hypothesis (>=3.44.24,<=6.31.6)", "pytest (>= [[package]] name = "identify" -version = "2.5.33" +version = "2.5.34" description = "File identification library for Python" optional = false python-versions = ">=3.8" files = [ - {file = "identify-2.5.33-py2.py3-none-any.whl", hash = "sha256:d40ce5fcd762817627670da8a7d8d8e65f24342d14539c59488dc603bf662e34"}, - {file = "identify-2.5.33.tar.gz", hash = "sha256:161558f9fe4559e1557e1bff323e8631f6a0e4837f7497767c1782832f16b62d"}, + {file = "identify-2.5.34-py2.py3-none-any.whl", hash = "sha256:a4316013779e433d08b96e5eabb7f641e6c7942e4ab5d4c509ebd2e7a8994aed"}, + {file = "identify-2.5.34.tar.gz", hash = "sha256:ee17bc9d499899bc9eaec1ac7bf2dc9eedd480db9d88b96d123d3b64a9d34f5d"}, ] [package.extras] @@ -1791,13 +1835,13 @@ files = [ [[package]] name = "netaddr" -version = "0.10.1" +version = "1.0.0" description = "A network address manipulation library for Python" optional = false python-versions = "*" files = [ - {file = "netaddr-0.10.1-py2.py3-none-any.whl", hash = "sha256:9822305b42ea1020d54fee322d43cee5622b044c07a1f0130b459bb467efcf88"}, - {file = "netaddr-0.10.1.tar.gz", hash = "sha256:f4da4222ca8c3f43c8e18a8263e5426c750a3a837fdfeccf74c68d0408eaa3bf"}, + {file = "netaddr-1.0.0-py3-none-any.whl", hash = "sha256:0dbf1263166c8daec9a6c990a9ae32804657963a1bba1387a755a4edf5ef93bc"}, + {file = "netaddr-1.0.0.tar.gz", hash = "sha256:eb046b55354e7a5bf801c04902ae923bd699190242d89b099e896f9b2724b4d3"}, ] [[package]] @@ -2099,6 +2143,142 @@ files = [ {file = "pycryptodomex-3.19.1.tar.gz", hash = "sha256:0b7154aff2272962355f8941fd514104a88cb29db2d8f43a29af900d6398eb1c"}, ] +[[package]] +name = "pydantic" +version = "2.5.3" +description = "Data validation using Python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-2.5.3-py3-none-any.whl", hash = "sha256:d0caf5954bee831b6bfe7e338c32b9e30c85dfe080c843680783ac2b631673b4"}, + {file = "pydantic-2.5.3.tar.gz", hash = "sha256:b3ef57c62535b0941697cce638c08900d87fcb67e29cfa99e8a68f747f393f7a"}, +] + +[package.dependencies] +annotated-types = ">=0.4.0" +pydantic-core = "2.14.6" +typing-extensions = ">=4.6.1" + +[package.extras] +email = ["email-validator (>=2.0.0)"] + +[[package]] +name = "pydantic-core" +version = "2.14.6" +description = "" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic_core-2.14.6-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:72f9a942d739f09cd42fffe5dc759928217649f070056f03c70df14f5770acf9"}, + {file = "pydantic_core-2.14.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6a31d98c0d69776c2576dda4b77b8e0c69ad08e8b539c25c7d0ca0dc19a50d6c"}, + {file = "pydantic_core-2.14.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5aa90562bc079c6c290f0512b21768967f9968e4cfea84ea4ff5af5d917016e4"}, + {file = "pydantic_core-2.14.6-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:370ffecb5316ed23b667d99ce4debe53ea664b99cc37bfa2af47bc769056d534"}, + {file = "pydantic_core-2.14.6-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f85f3843bdb1fe80e8c206fe6eed7a1caeae897e496542cee499c374a85c6e08"}, + {file = "pydantic_core-2.14.6-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9862bf828112e19685b76ca499b379338fd4c5c269d897e218b2ae8fcb80139d"}, + {file = "pydantic_core-2.14.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:036137b5ad0cb0004c75b579445a1efccd072387a36c7f217bb8efd1afbe5245"}, + {file = "pydantic_core-2.14.6-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:92879bce89f91f4b2416eba4429c7b5ca22c45ef4a499c39f0c5c69257522c7c"}, + {file = "pydantic_core-2.14.6-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0c08de15d50fa190d577e8591f0329a643eeaed696d7771760295998aca6bc66"}, + {file = "pydantic_core-2.14.6-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:36099c69f6b14fc2c49d7996cbf4f87ec4f0e66d1c74aa05228583225a07b590"}, + {file = "pydantic_core-2.14.6-cp310-none-win32.whl", hash = "sha256:7be719e4d2ae6c314f72844ba9d69e38dff342bc360379f7c8537c48e23034b7"}, + {file = "pydantic_core-2.14.6-cp310-none-win_amd64.whl", hash = "sha256:36fa402dcdc8ea7f1b0ddcf0df4254cc6b2e08f8cd80e7010d4c4ae6e86b2a87"}, + {file = "pydantic_core-2.14.6-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:dea7fcd62915fb150cdc373212141a30037e11b761fbced340e9db3379b892d4"}, + {file = "pydantic_core-2.14.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ffff855100bc066ff2cd3aa4a60bc9534661816b110f0243e59503ec2df38421"}, + {file = "pydantic_core-2.14.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b027c86c66b8627eb90e57aee1f526df77dc6d8b354ec498be9a757d513b92b"}, + {file = "pydantic_core-2.14.6-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:00b1087dabcee0b0ffd104f9f53d7d3eaddfaa314cdd6726143af6bc713aa27e"}, + {file = "pydantic_core-2.14.6-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:75ec284328b60a4e91010c1acade0c30584f28a1f345bc8f72fe8b9e46ec6a96"}, + {file = "pydantic_core-2.14.6-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7e1f4744eea1501404b20b0ac059ff7e3f96a97d3e3f48ce27a139e053bb370b"}, + {file = "pydantic_core-2.14.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b2602177668f89b38b9f84b7b3435d0a72511ddef45dc14446811759b82235a1"}, + {file = "pydantic_core-2.14.6-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6c8edaea3089bf908dd27da8f5d9e395c5b4dc092dbcce9b65e7156099b4b937"}, + {file = "pydantic_core-2.14.6-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:478e9e7b360dfec451daafe286998d4a1eeaecf6d69c427b834ae771cad4b622"}, + {file = "pydantic_core-2.14.6-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b6ca36c12a5120bad343eef193cc0122928c5c7466121da7c20f41160ba00ba2"}, + {file = "pydantic_core-2.14.6-cp311-none-win32.whl", hash = "sha256:2b8719037e570639e6b665a4050add43134d80b687288ba3ade18b22bbb29dd2"}, + {file = "pydantic_core-2.14.6-cp311-none-win_amd64.whl", hash = "sha256:78ee52ecc088c61cce32b2d30a826f929e1708f7b9247dc3b921aec367dc1b23"}, + {file = "pydantic_core-2.14.6-cp311-none-win_arm64.whl", hash = "sha256:a19b794f8fe6569472ff77602437ec4430f9b2b9ec7a1105cfd2232f9ba355e6"}, + {file = "pydantic_core-2.14.6-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:667aa2eac9cd0700af1ddb38b7b1ef246d8cf94c85637cbb03d7757ca4c3fdec"}, + {file = "pydantic_core-2.14.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cdee837710ef6b56ebd20245b83799fce40b265b3b406e51e8ccc5b85b9099b7"}, + {file = "pydantic_core-2.14.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c5bcf3414367e29f83fd66f7de64509a8fd2368b1edf4351e862910727d3e51"}, + {file = "pydantic_core-2.14.6-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:26a92ae76f75d1915806b77cf459811e772d8f71fd1e4339c99750f0e7f6324f"}, + {file = "pydantic_core-2.14.6-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a983cca5ed1dd9a35e9e42ebf9f278d344603bfcb174ff99a5815f953925140a"}, + {file = "pydantic_core-2.14.6-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cb92f9061657287eded380d7dc455bbf115430b3aa4741bdc662d02977e7d0af"}, + {file = "pydantic_core-2.14.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4ace1e220b078c8e48e82c081e35002038657e4b37d403ce940fa679e57113b"}, + {file = "pydantic_core-2.14.6-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ef633add81832f4b56d3b4c9408b43d530dfca29e68fb1b797dcb861a2c734cd"}, + {file = "pydantic_core-2.14.6-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7e90d6cc4aad2cc1f5e16ed56e46cebf4877c62403a311af20459c15da76fd91"}, + {file = "pydantic_core-2.14.6-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e8a5ac97ea521d7bde7621d86c30e86b798cdecd985723c4ed737a2aa9e77d0c"}, + {file = "pydantic_core-2.14.6-cp312-none-win32.whl", hash = "sha256:f27207e8ca3e5e021e2402ba942e5b4c629718e665c81b8b306f3c8b1ddbb786"}, + {file = "pydantic_core-2.14.6-cp312-none-win_amd64.whl", hash = "sha256:b3e5fe4538001bb82e2295b8d2a39356a84694c97cb73a566dc36328b9f83b40"}, + {file = "pydantic_core-2.14.6-cp312-none-win_arm64.whl", hash = "sha256:64634ccf9d671c6be242a664a33c4acf12882670b09b3f163cd00a24cffbd74e"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:24368e31be2c88bd69340fbfe741b405302993242ccb476c5c3ff48aeee1afe0"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:e33b0834f1cf779aa839975f9d8755a7c2420510c0fa1e9fa0497de77cd35d2c"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6af4b3f52cc65f8a0bc8b1cd9676f8c21ef3e9132f21fed250f6958bd7223bed"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d15687d7d7f40333bd8266f3814c591c2e2cd263fa2116e314f60d82086e353a"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:095b707bb287bfd534044166ab767bec70a9bba3175dcdc3371782175c14e43c"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:94fc0e6621e07d1e91c44e016cc0b189b48db053061cc22d6298a611de8071bb"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ce830e480f6774608dedfd4a90c42aac4a7af0a711f1b52f807130c2e434c06"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a306cdd2ad3a7d795d8e617a58c3a2ed0f76c8496fb7621b6cd514eb1532cae8"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:2f5fa187bde8524b1e37ba894db13aadd64faa884657473b03a019f625cee9a8"}, + {file = "pydantic_core-2.14.6-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:438027a975cc213a47c5d70672e0d29776082155cfae540c4e225716586be75e"}, + {file = "pydantic_core-2.14.6-cp37-none-win32.whl", hash = "sha256:f96ae96a060a8072ceff4cfde89d261837b4294a4f28b84a28765470d502ccc6"}, + {file = "pydantic_core-2.14.6-cp37-none-win_amd64.whl", hash = "sha256:e646c0e282e960345314f42f2cea5e0b5f56938c093541ea6dbf11aec2862391"}, + {file = "pydantic_core-2.14.6-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:db453f2da3f59a348f514cfbfeb042393b68720787bbef2b4c6068ea362c8149"}, + {file = "pydantic_core-2.14.6-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3860c62057acd95cc84044e758e47b18dcd8871a328ebc8ccdefd18b0d26a21b"}, + {file = "pydantic_core-2.14.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:36026d8f99c58d7044413e1b819a67ca0e0b8ebe0f25e775e6c3d1fabb3c38fb"}, + {file = "pydantic_core-2.14.6-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8ed1af8692bd8d2a29d702f1a2e6065416d76897d726e45a1775b1444f5928a7"}, + {file = "pydantic_core-2.14.6-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:314ccc4264ce7d854941231cf71b592e30d8d368a71e50197c905874feacc8a8"}, + {file = "pydantic_core-2.14.6-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:982487f8931067a32e72d40ab6b47b1628a9c5d344be7f1a4e668fb462d2da42"}, + {file = "pydantic_core-2.14.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dbe357bc4ddda078f79d2a36fc1dd0494a7f2fad83a0a684465b6f24b46fe80"}, + {file = "pydantic_core-2.14.6-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2f6ffc6701a0eb28648c845f4945a194dc7ab3c651f535b81793251e1185ac3d"}, + {file = "pydantic_core-2.14.6-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:7f5025db12fc6de7bc1104d826d5aee1d172f9ba6ca936bf6474c2148ac336c1"}, + {file = "pydantic_core-2.14.6-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:dab03ed811ed1c71d700ed08bde8431cf429bbe59e423394f0f4055f1ca0ea60"}, + {file = "pydantic_core-2.14.6-cp38-none-win32.whl", hash = "sha256:dfcbebdb3c4b6f739a91769aea5ed615023f3c88cb70df812849aef634c25fbe"}, + {file = "pydantic_core-2.14.6-cp38-none-win_amd64.whl", hash = "sha256:99b14dbea2fdb563d8b5a57c9badfcd72083f6006caf8e126b491519c7d64ca8"}, + {file = "pydantic_core-2.14.6-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:4ce8299b481bcb68e5c82002b96e411796b844d72b3e92a3fbedfe8e19813eab"}, + {file = "pydantic_core-2.14.6-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b9a9d92f10772d2a181b5ca339dee066ab7d1c9a34ae2421b2a52556e719756f"}, + {file = "pydantic_core-2.14.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd9e98b408384989ea4ab60206b8e100d8687da18b5c813c11e92fd8212a98e0"}, + {file = "pydantic_core-2.14.6-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4f86f1f318e56f5cbb282fe61eb84767aee743ebe32c7c0834690ebea50c0a6b"}, + {file = "pydantic_core-2.14.6-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86ce5fcfc3accf3a07a729779d0b86c5d0309a4764c897d86c11089be61da160"}, + {file = "pydantic_core-2.14.6-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3dcf1978be02153c6a31692d4fbcc2a3f1db9da36039ead23173bc256ee3b91b"}, + {file = "pydantic_core-2.14.6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eedf97be7bc3dbc8addcef4142f4b4164066df0c6f36397ae4aaed3eb187d8ab"}, + {file = "pydantic_core-2.14.6-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d5f916acf8afbcab6bacbb376ba7dc61f845367901ecd5e328fc4d4aef2fcab0"}, + {file = "pydantic_core-2.14.6-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:8a14c192c1d724c3acbfb3f10a958c55a2638391319ce8078cb36c02283959b9"}, + {file = "pydantic_core-2.14.6-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0348b1dc6b76041516e8a854ff95b21c55f5a411c3297d2ca52f5528e49d8411"}, + {file = "pydantic_core-2.14.6-cp39-none-win32.whl", hash = "sha256:de2a0645a923ba57c5527497daf8ec5df69c6eadf869e9cd46e86349146e5975"}, + {file = "pydantic_core-2.14.6-cp39-none-win_amd64.whl", hash = "sha256:aca48506a9c20f68ee61c87f2008f81f8ee99f8d7f0104bff3c47e2d148f89d9"}, + {file = "pydantic_core-2.14.6-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:d5c28525c19f5bb1e09511669bb57353d22b94cf8b65f3a8d141c389a55dec95"}, + {file = "pydantic_core-2.14.6-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:78d0768ee59baa3de0f4adac9e3748b4b1fffc52143caebddfd5ea2961595277"}, + {file = "pydantic_core-2.14.6-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b93785eadaef932e4fe9c6e12ba67beb1b3f1e5495631419c784ab87e975670"}, + {file = "pydantic_core-2.14.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a874f21f87c485310944b2b2734cd6d318765bcbb7515eead33af9641816506e"}, + {file = "pydantic_core-2.14.6-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b89f4477d915ea43b4ceea6756f63f0288941b6443a2b28c69004fe07fde0d0d"}, + {file = "pydantic_core-2.14.6-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:172de779e2a153d36ee690dbc49c6db568d7b33b18dc56b69a7514aecbcf380d"}, + {file = "pydantic_core-2.14.6-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:dfcebb950aa7e667ec226a442722134539e77c575f6cfaa423f24371bb8d2e94"}, + {file = "pydantic_core-2.14.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:55a23dcd98c858c0db44fc5c04fc7ed81c4b4d33c653a7c45ddaebf6563a2f66"}, + {file = "pydantic_core-2.14.6-pp37-pypy37_pp73-macosx_10_7_x86_64.whl", hash = "sha256:4241204e4b36ab5ae466ecec5c4c16527a054c69f99bba20f6f75232a6a534e2"}, + {file = "pydantic_core-2.14.6-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e574de99d735b3fc8364cba9912c2bec2da78775eba95cbb225ef7dda6acea24"}, + {file = "pydantic_core-2.14.6-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1302a54f87b5cd8528e4d6d1bf2133b6aa7c6122ff8e9dc5220fbc1e07bffebd"}, + {file = "pydantic_core-2.14.6-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f8e81e4b55930e5ffab4a68db1af431629cf2e4066dbdbfef65348b8ab804ea8"}, + {file = "pydantic_core-2.14.6-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c99462ffc538717b3e60151dfaf91125f637e801f5ab008f81c402f1dff0cd0f"}, + {file = "pydantic_core-2.14.6-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e4cf2d5829f6963a5483ec01578ee76d329eb5caf330ecd05b3edd697e7d768a"}, + {file = "pydantic_core-2.14.6-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:cf10b7d58ae4a1f07fccbf4a0a956d705356fea05fb4c70608bb6fa81d103cda"}, + {file = "pydantic_core-2.14.6-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:399ac0891c284fa8eb998bcfa323f2234858f5d2efca3950ae58c8f88830f145"}, + {file = "pydantic_core-2.14.6-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c6a5c79b28003543db3ba67d1df336f253a87d3112dac3a51b94f7d48e4c0e1"}, + {file = "pydantic_core-2.14.6-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:599c87d79cab2a6a2a9df4aefe0455e61e7d2aeede2f8577c1b7c0aec643ee8e"}, + {file = "pydantic_core-2.14.6-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:43e166ad47ba900f2542a80d83f9fc65fe99eb63ceec4debec160ae729824052"}, + {file = "pydantic_core-2.14.6-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:3a0b5db001b98e1c649dd55afa928e75aa4087e587b9524a4992316fa23c9fba"}, + {file = "pydantic_core-2.14.6-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:747265448cb57a9f37572a488a57d873fd96bf51e5bb7edb52cfb37124516da4"}, + {file = "pydantic_core-2.14.6-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:7ebe3416785f65c28f4f9441e916bfc8a54179c8dea73c23023f7086fa601c5d"}, + {file = "pydantic_core-2.14.6-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:86c963186ca5e50d5c8287b1d1c9d3f8f024cbe343d048c5bd282aec2d8641f2"}, + {file = "pydantic_core-2.14.6-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:e0641b506486f0b4cd1500a2a65740243e8670a2549bb02bc4556a83af84ae03"}, + {file = "pydantic_core-2.14.6-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:71d72ca5eaaa8d38c8df16b7deb1a2da4f650c41b58bb142f3fb75d5ad4a611f"}, + {file = "pydantic_core-2.14.6-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27e524624eace5c59af499cd97dc18bb201dc6a7a2da24bfc66ef151c69a5f2a"}, + {file = "pydantic_core-2.14.6-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a3dde6cac75e0b0902778978d3b1646ca9f438654395a362cb21d9ad34b24acf"}, + {file = "pydantic_core-2.14.6-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:00646784f6cd993b1e1c0e7b0fdcbccc375d539db95555477771c27555e3c556"}, + {file = "pydantic_core-2.14.6-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:23598acb8ccaa3d1d875ef3b35cb6376535095e9405d91a3d57a8c7db5d29341"}, + {file = "pydantic_core-2.14.6-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7f41533d7e3cf9520065f610b41ac1c76bc2161415955fbcead4981b22c7611e"}, + {file = "pydantic_core-2.14.6.tar.gz", hash = "sha256:1fd0c1d395372843fba13a51c28e3bb9d59bd7aebfeb17358ffaaa1e4dbbe948"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" + [[package]] name = "pyflakes" version = "3.2.0" @@ -2160,13 +2340,13 @@ hook-testing = ["execnet (>=1.5.0)", "psutil", "pytest (>=2.7.3)"] [[package]] name = "pyinstaller-hooks-contrib" -version = "2024.0" +version = "2024.1" description = "Community maintained hooks for PyInstaller" optional = false python-versions = ">=3.7" files = [ - {file = "pyinstaller-hooks-contrib-2024.0.tar.gz", hash = "sha256:a7118c1a5c9788595e5c43ad058a7a5b7b6d59e1eceb42362f6ec1f0b61986b0"}, - {file = "pyinstaller_hooks_contrib-2024.0-py2.py3-none-any.whl", hash = "sha256:469b5690df53223e2e8abffb2e44d6ee596e7d79d4b1eed9465123b67439875a"}, + {file = "pyinstaller-hooks-contrib-2024.1.tar.gz", hash = "sha256:51a51ea9e1ae6bd5ffa7ec45eba7579624bf4f2472ff56dba0edc186f6ed46a6"}, + {file = "pyinstaller_hooks_contrib-2024.1-py2.py3-none-any.whl", hash = "sha256:131494f9cfce190aaa66ed82e82c78b2723d1720ce64d012fbaf938f4ab01d35"}, ] [package.dependencies] @@ -2734,18 +2914,18 @@ tornado = ["tornado (>=5)"] [[package]] name = "setuptools" -version = "69.0.3" +version = "69.1.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-69.0.3-py3-none-any.whl", hash = "sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05"}, - {file = "setuptools-69.0.3.tar.gz", hash = "sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78"}, + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, ] [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] @@ -2770,6 +2950,17 @@ files = [ {file = "smmap-5.0.1.tar.gz", hash = "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62"}, ] +[[package]] +name = "sniffio" +version = "1.3.0" +description = "Sniff out which async library your code is running under" +optional = false +python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"}, + {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, +] + [[package]] name = "staking_deposit" version = "2.4.0" @@ -2785,6 +2976,23 @@ url = "https://github.com/ethereum/staking-deposit-cli.git" reference = "v2.4.0" resolved_reference = "ef89710443814331aa2f592067dc4d6995cc4f6e" +[[package]] +name = "starlette" +version = "0.36.2" +description = "The little ASGI library that shines." +optional = false +python-versions = ">=3.8" +files = [ + {file = "starlette-0.36.2-py3-none-any.whl", hash = "sha256:e53e086e620ba715f0c187daca92927b722484d148e70921f0de075936119536"}, + {file = "starlette-0.36.2.tar.gz", hash = "sha256:4134757b950f027c8f16028ec81787751bb44145df17c8b0fa84087b9851b42d"}, +] + +[package.dependencies] +anyio = ">=3.4.0,<5" + +[package.extras] +full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.7)", "pyyaml"] + [[package]] name = "stevedore" version = "5.1.0" @@ -2908,6 +3116,25 @@ h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] +[[package]] +name = "uvicorn" +version = "0.27.0" +description = "The lightning-fast ASGI server." +optional = false +python-versions = ">=3.8" +files = [ + {file = "uvicorn-0.27.0-py3-none-any.whl", hash = "sha256:890b00f6c537d58695d3bb1f28e23db9d9e7a17cbcc76d7457c499935f933e24"}, + {file = "uvicorn-0.27.0.tar.gz", hash = "sha256:c855578045d45625fd027367f7653d249f7c49f9361ba15cf9624186b26b8eb6"}, +] + +[package.dependencies] +click = ">=7.0" +h11 = ">=0.8" +typing-extensions = {version = ">=4.0", markers = "python_version < \"3.11\""} + +[package.extras] +standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.4)"] + [[package]] name = "varint" version = "1.0.2" @@ -3160,4 +3387,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.11" -content-hash = "146b3130ab7834f79d12739efacd04d621e020b94fb892aef66d2028ba7a71af" +content-hash = "b3b13c3de1a79629250ebae7e1c1a5c4fc1227ab55d0e8e4555b1c2b0e35af0d" diff --git a/pyproject.toml b/pyproject.toml index 7b23496c..cb6a171f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,9 @@ psycopg2 = "==2.9.9" pyyaml = "==6.0.1" aiohttp = "==3.9.3" python-json-logger = "==2.0.7" +starlette = "==0.36.2" +uvicorn = "==0.27.0" +pydantic = "==2.5.3" [tool.poetry.group.dev.dependencies] pylint = "==3.0.1" diff --git a/src/api.py b/src/api.py new file mode 100644 index 00000000..c48fa264 --- /dev/null +++ b/src/api.py @@ -0,0 +1,11 @@ +from starlette.applications import Starlette +from starlette.routing import Route + +from src.validators.api.endpoints import get_validators, submit_validators + +app = Starlette( + routes=[ + Route('/validators', get_validators, methods=['GET']), + Route('/validators', submit_validators, methods=['POST']), + ] +) diff --git a/src/commands/start.py b/src/commands/start.py index 4f4cb84c..24982a2d 100644 --- a/src/commands/start.py +++ b/src/commands/start.py @@ -4,15 +4,10 @@ import click from eth_typing import ChecksumAddress -from sw_utils import EventScanner, InterruptHandler -import src -from src.common.consensus import get_chain_finalized_head -from src.common.execution import WalletTask -from src.common.logging import LOG_LEVELS, setup_logging -from src.common.metrics import MetricsTask, metrics_server -from src.common.startup_check import startup_checks -from src.common.utils import get_build_version, log_verbose +from src.commands.start_base import start_base +from src.common.logging import LOG_LEVELS +from src.common.utils import log_verbose from src.common.validators import validate_eth_address from src.common.vault_config import VaultConfig from src.config.settings import ( @@ -24,13 +19,6 @@ LOG_PLAIN, settings, ) -from src.exits.tasks import ExitSignatureTask -from src.harvest.tasks import HarvestTask -from src.validators.database import NetworkValidatorCrud -from src.validators.execution import NetworkValidatorsProcessor -from src.validators.keystores.load import load_keystore -from src.validators.tasks import ValidatorsTask, load_genesis_validators -from src.validators.utils import load_deposit_data logger = logging.getLogger(__name__) @@ -264,76 +252,6 @@ def start( ) try: - asyncio.run(main()) + asyncio.run(start_base()) except Exception as e: log_verbose(e) - - -async def main() -> None: - setup_logging() - setup_sentry() - log_start() - - await startup_checks() - - NetworkValidatorCrud().setup() - - # load network validators from ipfs dump - await load_genesis_validators() - - # load keystore - keystore = await load_keystore() - - # load deposit data - deposit_data = load_deposit_data(settings.vault, settings.deposit_data_file) - logger.info('Loaded deposit data file %s', settings.deposit_data_file) - # start operator tasks - - # periodically scan network validator updates - network_validators_processor = NetworkValidatorsProcessor() - network_validators_scanner = EventScanner(network_validators_processor) - - logger.info('Syncing network validator events...') - chain_state = await get_chain_finalized_head() - await network_validators_scanner.process_new_events(chain_state.execution_block) - - if settings.enable_metrics: - await metrics_server() - - logger.info('Started operator service') - with InterruptHandler() as interrupt_handler: - tasks = [ - ValidatorsTask( - keystore=keystore, - deposit_data=deposit_data, - ).run(interrupt_handler), - ExitSignatureTask( - keystore=keystore, - ).run(interrupt_handler), - MetricsTask().run(interrupt_handler), - WalletTask().run(interrupt_handler), - ] - if settings.harvest_vault: - tasks.append(HarvestTask().run(interrupt_handler)) - - await asyncio.gather(*tasks) - - -def log_start() -> None: - build = get_build_version() - start_str = 'Starting operator service' - - if build: - logger.info('%s, version %s, build %s', start_str, src.__version__, build) - else: - logger.info('%s, version %s', start_str, src.__version__) - - -def setup_sentry(): - if settings.sentry_dsn: - # pylint: disable-next=import-outside-toplevel - import sentry_sdk - - sentry_sdk.init(settings.sentry_dsn, traces_sample_rate=0.1) - sentry_sdk.set_tag('network', settings.network) - sentry_sdk.set_tag('vault', settings.vault) diff --git a/src/commands/start_api.py b/src/commands/start_api.py new file mode 100644 index 00000000..7d80c485 --- /dev/null +++ b/src/commands/start_api.py @@ -0,0 +1,227 @@ +import asyncio +import logging +from pathlib import Path + +import click +from eth_typing import ChecksumAddress + +import src.validators.api.endpoints # noqa # pylint:disable=unused-import +from src.commands.start_base import start_base +from src.common.logging import LOG_LEVELS +from src.common.utils import log_verbose +from src.common.validators import validate_eth_address +from src.common.vault_config import VaultConfig +from src.config.settings import ( + AVAILABLE_NETWORKS, + DEFAULT_API_HOST, + DEFAULT_API_PORT, + DEFAULT_MAX_FEE_PER_GAS_GWEI, + DEFAULT_METRICS_HOST, + DEFAULT_METRICS_PORT, + LOG_FORMATS, + LOG_PLAIN, + settings, +) +from src.validators.typings import ValidatorsRegistrationMode + +logger = logging.getLogger(__name__) + + +@click.option( + '--data-dir', + default=str(Path.home() / '.stakewise'), + envvar='DATA_DIR', + help='Path where the vault data will be placed. Default is ~/.stakewise.', + type=click.Path(exists=True, file_okay=False, dir_okay=True), +) +@click.option( + '--database-dir', + type=click.Path(exists=True, file_okay=False, dir_okay=True), + envvar='DATABASE_DIR', + help='The directory where the database will be created or read from. ' + 'Default is ~/.stakewise/.', +) +@click.option( + '--max-fee-per-gas-gwei', + type=int, + envvar='MAX_FEE_PER_GAS_GWEI', + help=f'Maximum fee per gas limit for transactions. ' + f'Default is {DEFAULT_MAX_FEE_PER_GAS_GWEI} Gwei.', + default=DEFAULT_MAX_FEE_PER_GAS_GWEI, +) +@click.option( + '--hot-wallet-password-file', + type=click.Path(exists=True, file_okay=True, dir_okay=False), + envvar='HOT_WALLET_PASSWORD_FILE', + help='Absolute path to the hot wallet password file. ' + 'Default is the file generated with "create-wallet" command.', +) +@click.option( + '--hot-wallet-file', + type=click.Path(exists=True, file_okay=True, dir_okay=False), + envvar='HOT_WALLET_FILE', + help='Absolute path to the hot wallet. ' + 'Default is the file generated with "create-wallet" command.', +) +@click.option( + '--deposit-data-file', + type=click.Path(exists=True, file_okay=True, dir_okay=False), + envvar='DEPOSIT_DATA_FILE', + help='Path to the deposit_data.json file. ' + 'Default is the file generated with "create-keys" command.', +) +@click.option( + '--network', + type=click.Choice( + AVAILABLE_NETWORKS, + case_sensitive=False, + ), + envvar='NETWORK', + help='The network of the vault. Default is the network specified at "init" command.', +) +@click.option( + '--enable-metrics', + is_flag=True, + envvar='ENABLE_METRICS', + help='Whether to enable metrics server. Disabled by default.', +) +@click.option( + '--metrics-host', + type=str, + help=f'The prometheus metrics host. Default is {DEFAULT_METRICS_HOST}.', + envvar='METRICS_HOST', + default=DEFAULT_METRICS_HOST, +) +@click.option( + '--metrics-port', + type=int, + help=f'The prometheus metrics port. Default is {DEFAULT_METRICS_PORT}.', + envvar='METRICS_PORT', + default=DEFAULT_METRICS_PORT, +) +@click.option( + '-v', + '--verbose', + help='Enable debug mode. Default is false.', + envvar='VERBOSE', + is_flag=True, +) +@click.option( + '--harvest-vault', + is_flag=True, + envvar='HARVEST_VAULT', + help='Whether to submit vault harvest transactions. Default is false.', +) +@click.option( + '--execution-endpoints', + type=str, + envvar='EXECUTION_ENDPOINTS', + prompt='Enter comma separated list of API endpoints for execution nodes', + help='Comma separated list of API endpoints for execution nodes.', +) +@click.option( + '--consensus-endpoints', + type=str, + envvar='CONSENSUS_ENDPOINTS', + prompt='Enter comma separated list of API endpoints for consensus nodes', + help='Comma separated list of API endpoints for consensus nodes.', +) +@click.option( + '--vault', + type=ChecksumAddress, + callback=validate_eth_address, + envvar='VAULT', + prompt='Enter the vault address', + help='Address of the vault to register validators for.', +) +@click.option( + '--log-format', + type=click.Choice( + LOG_FORMATS, + case_sensitive=False, + ), + default=LOG_PLAIN, + envvar='LOG_FORMAT', + help='The log record format. Can be "plain" or "json".', +) +@click.option( + '--log-level', + type=click.Choice( + LOG_LEVELS, + case_sensitive=False, + ), + default='INFO', + envvar='LOG_LEVEL', + help='The log level.', +) +@click.option( + '--api-host', + type=str, + help=f'API host. Default is {DEFAULT_API_HOST}.', + envvar='API_HOST', + default=DEFAULT_API_HOST, +) +@click.option( + '--api-port', + type=int, + help=f'API port. Default is {DEFAULT_API_PORT}.', + envvar='API_PORT', + default=DEFAULT_API_PORT, +) +@click.command(help='Start operator service') +# pylint: disable-next=too-many-arguments,too-many-locals +def start_api( + vault: ChecksumAddress, + consensus_endpoints: str, + execution_endpoints: str, + harvest_vault: bool, + verbose: bool, + enable_metrics: bool, + metrics_host: str, + metrics_port: int, + data_dir: str, + log_level: str, + log_format: str, + network: str | None, + deposit_data_file: str | None, + hot_wallet_file: str | None, + hot_wallet_password_file: str | None, + max_fee_per_gas_gwei: int, + database_dir: str | None, + api_host: str, + api_port: int, +) -> None: + vault_config = VaultConfig(vault, Path(data_dir)) + if network is None: + vault_config.load() + network = vault_config.network + + validators_registration_mode = ValidatorsRegistrationMode.API + + settings.set( + vault=vault, + vault_dir=vault_config.vault_dir, + consensus_endpoints=consensus_endpoints, + execution_endpoints=execution_endpoints, + harvest_vault=harvest_vault, + verbose=verbose, + enable_metrics=enable_metrics, + metrics_host=metrics_host, + metrics_port=metrics_port, + network=network, + deposit_data_file=deposit_data_file, + hot_wallet_file=hot_wallet_file, + hot_wallet_password_file=hot_wallet_password_file, + max_fee_per_gas_gwei=max_fee_per_gas_gwei, + database_dir=database_dir, + log_level=log_level, + log_format=log_format, + api_host=api_host, + api_port=api_port, + validators_registration_mode=validators_registration_mode, + ) + + try: + asyncio.run(start_base()) + except Exception as e: + log_verbose(e) diff --git a/src/commands/start_base.py b/src/commands/start_base.py new file mode 100644 index 00000000..a9e78e54 --- /dev/null +++ b/src/commands/start_base.py @@ -0,0 +1,121 @@ +import asyncio +import logging + +import uvicorn +from sw_utils import EventScanner, InterruptHandler + +import src +from src.api import app as api_app +from src.common.consensus import get_chain_finalized_head +from src.common.execution import WalletTask, update_oracles_cache +from src.common.logging import setup_logging +from src.common.metrics import MetricsTask, metrics_server +from src.common.startup_check import startup_checks +from src.common.utils import get_build_version +from src.config.settings import settings +from src.exits.tasks import ExitSignatureTask +from src.harvest.tasks import HarvestTask +from src.validators.database import NetworkValidatorCrud +from src.validators.execution import NetworkValidatorsProcessor +from src.validators.keystores.base import BaseKeystore +from src.validators.keystores.load import load_keystore +from src.validators.tasks import ValidatorsTask, load_genesis_validators +from src.validators.typings import ValidatorsRegistrationMode +from src.validators.utils import load_deposit_data + +logger = logging.getLogger(__name__) + + +async def start_base() -> None: + setup_logging() + setup_sentry() + log_start() + + if not settings.skip_startup_checks: + await startup_checks() + + NetworkValidatorCrud().setup() + + # load network validators from ipfs dump + await load_genesis_validators() + + # load keystore + keystore: BaseKeystore | None = None + if settings.validators_registration_mode == ValidatorsRegistrationMode.AUTO: + keystore = await load_keystore() + + # load deposit data + deposit_data = load_deposit_data(settings.vault, settings.deposit_data_file) + logger.info('Loaded deposit data file %s', settings.deposit_data_file) + # start operator tasks + + # periodically scan network validator updates + network_validators_processor = NetworkValidatorsProcessor() + network_validators_scanner = EventScanner(network_validators_processor) + + logger.info('Syncing network validator events...') + chain_state = await get_chain_finalized_head() + await network_validators_scanner.process_new_events(chain_state.execution_block) + + logger.info('Updating oracles cache...') + await update_oracles_cache() + + if settings.validators_registration_mode == ValidatorsRegistrationMode.API: + logger.info('Starting api server') + api_app.state.deposit_data = deposit_data + + config = uvicorn.Config( + api_app, + host=settings.api_host, + port=settings.api_port, + log_config=None, + ) + server = UvicornServerWithoutSignals(config) + asyncio.create_task(server.serve()) + + if settings.enable_metrics: + await metrics_server() + + logger.info('Started operator service') + with InterruptHandler() as interrupt_handler: + tasks = [ + ValidatorsTask( + keystore=keystore, + deposit_data=deposit_data, + ).run(interrupt_handler), + ExitSignatureTask( + keystore=keystore, + ).run(interrupt_handler), + MetricsTask().run(interrupt_handler), + WalletTask().run(interrupt_handler), + ] + if settings.harvest_vault: + tasks.append(HarvestTask().run(interrupt_handler)) + + await asyncio.gather(*tasks) + + +class UvicornServerWithoutSignals(uvicorn.Server): + def install_signal_handlers(self) -> None: + # Manage signals in command, not in Uvicorn + pass + + +def log_start() -> None: + build = get_build_version() + start_str = 'Starting operator service' + + if build: + logger.info('%s, version %s, build %s', start_str, src.__version__, build) + else: + logger.info('%s, version %s', start_str, src.__version__) + + +def setup_sentry(): + if settings.sentry_dsn: + # pylint: disable-next=import-outside-toplevel + import sentry_sdk + + sentry_sdk.init(settings.sentry_dsn, traces_sample_rate=0.1) + sentry_sdk.set_tag('network', settings.network) + sentry_sdk.set_tag('vault', settings.vault) diff --git a/src/commands/validators_exit.py b/src/commands/validators_exit.py index 392b6cef..e290637b 100644 --- a/src/commands/validators_exit.py +++ b/src/commands/validators_exit.py @@ -156,6 +156,7 @@ def validators_exit( async def main(count: int | None) -> None: setup_logging() keystore = await load_keystore() + validators_exits = await _get_validators_exits(keystore=keystore) if not validators_exits: raise click.ClickException('There are no active validators.') @@ -177,8 +178,6 @@ async def main(count: int | None) -> None: exit_signature = await keystore.get_exit_signature( validator_index=validator_exit.index, public_key=validator_exit.public_key, - network=settings.network, - fork=settings.network_config.SHAPELLA_FORK, ) try: await consensus_client.submit_voluntary_exit( diff --git a/src/common/metrics.py b/src/common/metrics.py index ff7fc76e..baa92a56 100644 --- a/src/common/metrics.py +++ b/src/common/metrics.py @@ -1,3 +1,5 @@ +import logging + from prometheus_client import Gauge, Info, start_http_server import src @@ -27,8 +29,11 @@ def set_app_version(self): metrics = Metrics() metrics.set_app_version() +logger = logging.getLogger(__name__) + async def metrics_server() -> None: + logger.info('Starting metrics server') start_http_server(settings.metrics_port, settings.metrics_host) diff --git a/src/config/settings.py b/src/config/settings.py index a3e6adfd..d0bc9305 100644 --- a/src/config/settings.py +++ b/src/config/settings.py @@ -6,13 +6,18 @@ from web3.types import ChecksumAddress from src.config.networks import HOLESKY, MAINNET, NETWORKS, NetworkConfig +from src.validators.typings import ValidatorsRegistrationMode DATA_DIR = Path.home() / '.stakewise' DEFAULT_MAX_FEE_PER_GAS_GWEI = 100 + DEFAULT_METRICS_HOST = '127.0.0.1' DEFAULT_METRICS_PORT = 9100 +DEFAULT_API_HOST = '127.0.0.1' +DEFAULT_API_PORT = 8000 + class Singleton(type): _instances: dict = {} @@ -67,6 +72,11 @@ class Settings(metaclass=Singleton): sentry_dsn: str pool_size: int | None + api_host: str + api_port: int + validators_registration_mode: ValidatorsRegistrationMode + skip_startup_checks: bool + # pylint: disable-next=too-many-arguments,too-many-locals def set( self, @@ -94,6 +104,9 @@ def set( log_level: str | None = None, log_format: str | None = None, pool_size: int | None = None, + api_host: str = DEFAULT_API_HOST, + api_port: int = DEFAULT_API_PORT, + validators_registration_mode: ValidatorsRegistrationMode = ValidatorsRegistrationMode.AUTO, ) -> None: self.vault = Web3.to_checksum_address(vault) vault_dir.mkdir(parents=True, exist_ok=True) @@ -186,6 +199,11 @@ def set( self.consensus_retry_timeout = decouple_config( 'CONSENSUS_RETRY_TIMEOUT', default=120, cast=int ) + self.api_host = api_host + self.api_port = api_port + self.validators_registration_mode = validators_registration_mode + + self.skip_startup_checks = decouple_config('SKIP_STARTUP_CHECKS', default=False, cast=bool) @property def keystore_cls_str(self) -> str: diff --git a/src/exits/tasks.py b/src/exits/tasks.py index 68c94678..95269ca3 100644 --- a/src/exits/tasks.py +++ b/src/exits/tasks.py @@ -24,15 +24,19 @@ send_signature_rotation_requests, ) from src.validators.keystores.base import BaseKeystore +from src.validators.signing.common import get_encrypted_exit_signature_shards logger = logging.getLogger(__name__) class ExitSignatureTask(BaseTask): - def __init__(self, keystore: BaseKeystore): + def __init__(self, keystore: BaseKeystore | None): self.keystore = keystore async def process_block(self) -> None: + if self.keystore is None: + return + oracles = await get_oracles() update_block = await _fetch_last_update_block() if update_block and not await is_block_finalized(update_block): @@ -190,11 +194,11 @@ async def _get_oracles_request( break if public_key in keystore: - shards = await keystore.get_exit_signature_shards( - validator_index=validator_index, + shards = await get_encrypted_exit_signature_shards( + keystore=keystore, public_key=public_key, + validator_index=validator_index, oracles=oracles, - fork=settings.network_config.SHAPELLA_FORK, ) else: failed_indexes.append(validator_index) diff --git a/src/exits/tests/test_tasks.py b/src/exits/tests/test_tasks.py index da2cb7b6..c3365be7 100644 --- a/src/exits/tests/test_tasks.py +++ b/src/exits/tests/test_tasks.py @@ -13,7 +13,6 @@ from src.exits.tasks import _get_oracles_request from src.validators.keystores.local import Keys, LocalKeystore from src.validators.keystores.remote import RemoteSignerKeystore -from src.validators.typings import ExitSignatureShards @pytest.mark.usefixtures('fake_settings') @@ -65,13 +64,6 @@ async def test_remote_signer( epoch=1, ), ), - mock.patch( - 'src.exits.tasks.BaseKeystore.get_exit_signature_shards', - return_value=ExitSignatureShards( - public_keys=[], - exit_signatures=[], - ), - ), ): validators = { randint(0, int(1e6)): pubkey for pubkey in remote_signer_keystore.public_keys diff --git a/src/main.py b/src/main.py index 74f5fc32..5991224e 100644 --- a/src/main.py +++ b/src/main.py @@ -16,6 +16,7 @@ from src.commands.recover import recover from src.commands.remote_signer_setup import remote_signer_setup from src.commands.start import start +from src.commands.start_api import start_api from src.commands.validators_exit import validators_exit from src.common.utils import get_build_version from src.remote_db.commands import remote_db_group @@ -39,6 +40,7 @@ def cli() -> None: cli.add_command(merge_deposit_data) cli.add_command(validators_exit) cli.add_command(start) +cli.add_command(start_api) cli.add_command(recover) cli.add_command(get_validators_root) cli.add_command(import_genesis_keys) diff --git a/src/validators/api/__init__.py b/src/validators/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/validators/api/endpoints.py b/src/validators/api/endpoints.py new file mode 100644 index 00000000..195e7853 --- /dev/null +++ b/src/validators/api/endpoints.py @@ -0,0 +1,128 @@ +import asyncio +import json +from typing import Any + +from pydantic import TypeAdapter, ValidationError +from starlette.requests import Request +from starlette.responses import JSONResponse, Response +from starlette.status import HTTP_400_BAD_REQUEST +from sw_utils import is_valid_exit_signature +from web3 import Web3 + +from src.config.settings import settings +from src.validators.api.schemas import ValidatorRegistrationRequest +from src.validators.database import NetworkValidatorCrud +from src.validators.execution import get_latest_network_validator_public_keys +from src.validators.tasks import ( + get_available_validators_for_registration, + pending_validator_registrations, + register_and_remove_pending_validators, +) +from src.validators.typings import Validator, ValidatorsRegistrationMode + + +async def get_validators(request: Request) -> Response: + deposit_data = request.app.state.deposit_data + + validators = await get_available_validators_for_registration( + keystore=None, deposit_data=deposit_data, run_check_deposit_data_root=False + ) + validators = [v for v in validators if v.public_key not in pending_validator_registrations] + + # get next validator index for exit signature + latest_public_keys = await get_latest_network_validator_public_keys() + next_validator_index = NetworkValidatorCrud().get_next_validator_index(list(latest_public_keys)) + + return JSONResponse( + [ + {'public_key': validator.public_key, 'index': index} + for index, validator in enumerate(validators, next_validator_index) + ] + ) + + +async def submit_validators(request: Request) -> Response: + if settings.validators_registration_mode != ValidatorsRegistrationMode.API: + return JSONResponse( + {'error': 'validators registration mode must be "API"'}, + status_code=HTTP_400_BAD_REQUEST, + ) + + try: + payload = await request.json() + except json.JSONDecodeError: + return JSONResponse({'error': 'invalid json'}, status_code=HTTP_400_BAD_REQUEST) + + adapter = TypeAdapter(list[ValidatorRegistrationRequest]) + try: + registration_requests: list[ValidatorRegistrationRequest] = adapter.validate_python(payload) + except ValidationError as e: + return JSONResponse({'error': json.loads(e.json())}, status_code=HTTP_400_BAD_REQUEST) + + if not registration_requests: + return JSONResponse({'error': 'invalid validators'}, status_code=HTTP_400_BAD_REQUEST) + + deposit_data = request.app.state.deposit_data + + validators = await get_available_validators_for_registration( + keystore=None, deposit_data=deposit_data + ) + + # There may be lag between GET and POST requests. + # During this time new assets may be staked into the vault. + # In this case validators list will be longer than requests list. + # + # If someone unstakes funds then validators list may become shorter + # than requests list + # + common_length = min(len(validators), len(registration_requests)) + registration_requests = registration_requests[:common_length] + + error = await _validate_registration_requests(registration_requests, validators) + if error is not None: + return JSONResponse({'error': error}, status_code=HTTP_400_BAD_REQUEST) + + for validator, registration_request in zip(validators, registration_requests): + validator.exit_signature = registration_request.exit_signature + + pending_validator_registrations.extend([v.public_key for v in validators]) + asyncio.create_task( + register_and_remove_pending_validators( + keystore=None, deposit_data=deposit_data, validators=validators + ) + ) + + return JSONResponse({}) + + +async def _validate_registration_requests( + registration_requests: list[ValidatorRegistrationRequest], validators: list[Validator] +) -> Any: + """ + Business logic validation + :return: error + """ + if len(validators) < len(registration_requests): + return 'invalid validators length' + + # get next validator index for exit signature + latest_public_keys = await get_latest_network_validator_public_keys() + next_validator_index = NetworkValidatorCrud().get_next_validator_index(list(latest_public_keys)) + + # validate public keys and exit signatures + for registration_request, (validator_index, validator) in zip( + registration_requests, enumerate(validators, next_validator_index) + ): + if validator.public_key != Web3.to_hex(registration_request.public_key): + return 'invalid validators public_key' + + if not is_valid_exit_signature( + validator_index, + registration_request.public_key, + registration_request.exit_signature, + settings.network_config.GENESIS_VALIDATORS_ROOT, + settings.network_config.SHAPELLA_FORK, + ): + return 'invalid validators exit_signature' + + return None diff --git a/src/validators/api/fields.py b/src/validators/api/fields.py new file mode 100644 index 00000000..cb7841b3 --- /dev/null +++ b/src/validators/api/fields.py @@ -0,0 +1,8 @@ +from eth_typing import BLSPubkey, BLSSignature +from pydantic import BeforeValidator +from typing_extensions import Annotated + +from src.validators.api.validators import to_bls_pubkey, to_bls_signature + +BLSPubkeyField = Annotated[BLSPubkey, BeforeValidator(to_bls_pubkey)] +BLSSignatureField = Annotated[BLSSignature, BeforeValidator(to_bls_signature)] diff --git a/src/validators/api/schemas.py b/src/validators/api/schemas.py new file mode 100644 index 00000000..c8a31452 --- /dev/null +++ b/src/validators/api/schemas.py @@ -0,0 +1,8 @@ +from pydantic import BaseModel + +from src.validators.api.fields import BLSPubkeyField, BLSSignatureField + + +class ValidatorRegistrationRequest(BaseModel): + public_key: BLSPubkeyField + exit_signature: BLSSignatureField diff --git a/src/validators/api/validators.py b/src/validators/api/validators.py new file mode 100644 index 00000000..a2845f55 --- /dev/null +++ b/src/validators/api/validators.py @@ -0,0 +1,28 @@ +from typing import Any + +from eth_typing import BLSPubkey, BLSSignature +from py_ecc.bls.ciphersuites import G2ProofOfPossession +from web3 import Web3 + + +def to_bls_pubkey(v: Any) -> BLSPubkey: + try: + pubkey = BLSPubkey(Web3.to_bytes(hexstr=v)) + if G2ProofOfPossession.KeyValidate(pubkey): + return pubkey + except Exception: # nosec + pass + + raise ValueError('invalid public key') + + +def to_bls_signature(v: Any) -> BLSSignature: + try: + signature = BLSSignature(Web3.to_bytes(hexstr=v)) + # pylint: disable=protected-access + if G2ProofOfPossession._is_valid_signature(signature): + return signature + except Exception: # nosec + pass + + raise ValueError('invalid bls signature') diff --git a/src/validators/execution.py b/src/validators/execution.py index 78125dc1..7359595b 100644 --- a/src/validators/execution.py +++ b/src/validators/execution.py @@ -160,12 +160,14 @@ async def check_deposit_data_root(deposit_data_root: str) -> None: async def get_available_validators( - keystore: BaseKeystore, + keystore: BaseKeystore | None, deposit_data: DepositData, count: int, + run_check_deposit_data_root: bool = True, ) -> list[Validator]: """Fetches vault's available validators.""" - await check_deposit_data_root(deposit_data.tree.root) + if run_check_deposit_data_root: + await check_deposit_data_root(deposit_data.tree.root) start_index = await vault_contract.get_validators_index() validators: list[Validator] = [] @@ -176,7 +178,7 @@ async def get_available_validators( validator = deposit_data.validators[i] except IndexError: break - if validator.public_key not in keystore: + if keystore and validator.public_key not in keystore: logger.warning( 'Cannot find validator with public key %s in keystores.', validator.public_key, diff --git a/src/validators/keystores/base.py b/src/validators/keystores/base.py index 4016b801..8fb8868d 100644 --- a/src/validators/keystores/base.py +++ b/src/validators/keystores/base.py @@ -3,9 +3,6 @@ from eth_typing import BLSSignature, HexStr from sw_utils.typings import ConsensusFork -from src.common.typings import Oracles -from src.validators.typings import ExitSignatureShards - class BaseKeystore(abc.ABC): @staticmethod @@ -25,15 +22,9 @@ def __contains__(self, public_key: HexStr) -> bool: def __len__(self) -> int: raise NotImplementedError - @abc.abstractmethod - async def get_exit_signature_shards( - self, validator_index: int, public_key: HexStr, oracles: Oracles, fork: ConsensusFork - ) -> ExitSignatureShards: - raise NotImplementedError - @abc.abstractmethod async def get_exit_signature( - self, validator_index: int, public_key: HexStr, network: str, fork: ConsensusFork + self, validator_index: int, public_key: HexStr, fork: ConsensusFork | None = None ) -> BLSSignature: raise NotImplementedError diff --git a/src/validators/keystores/local.py b/src/validators/keystores/local.py index 1d05ea44..92a1b5aa 100644 --- a/src/validators/keystores/local.py +++ b/src/validators/keystores/local.py @@ -13,13 +13,10 @@ from sw_utils.typings import ConsensusFork from web3 import Web3 -from src.common.typings import Oracles -from src.config.settings import NETWORKS, settings +from src.config.settings import settings from src.validators.exceptions import KeystoreException from src.validators.keystores.base import BaseKeystore -from src.validators.signing.common import encrypt_signature -from src.validators.signing.key_shares import private_key_to_private_key_shares -from src.validators.typings import BLSPrivkey, ExitSignatureShards +from src.validators.typings import BLSPrivkey logger = logging.getLogger(__name__) @@ -80,40 +77,16 @@ def __contains__(self, public_key): def __len__(self) -> int: return len(self.keys) - async def get_exit_signature_shards( - self, validator_index: int, public_key: HexStr, oracles: Oracles, fork: ConsensusFork - ) -> ExitSignatureShards: - """Generates exit signature shards and encrypts them with oracles' public keys.""" - message = get_exit_message_signing_root( - validator_index=validator_index, - genesis_validators_root=settings.network_config.GENESIS_VALIDATORS_ROOT, - fork=fork, - ) - - private_key_shares = private_key_to_private_key_shares( - private_key=self.keys[public_key], - threshold=oracles.exit_signature_recover_threshold, - total=len(oracles.public_keys), - ) - exit_signature_shards: list[HexStr] = [] - for bls_priv_key, oracle_pubkey in zip(private_key_shares, oracles.public_keys): - exit_signature_shards.append( - encrypt_signature(oracle_pubkey, bls.Sign(bls_priv_key, message)) - ) - - return ExitSignatureShards( - public_keys=[Web3.to_hex(bls.SkToPk(priv_key)) for priv_key in private_key_shares], - exit_signatures=exit_signature_shards, - ) - async def get_exit_signature( - self, validator_index: int, public_key: HexStr, network: str, fork: ConsensusFork + self, validator_index: int, public_key: HexStr, fork: ConsensusFork | None = None ) -> BLSSignature: + fork = fork or settings.network_config.SHAPELLA_FORK + private_key = self.keys[public_key] message = get_exit_message_signing_root( validator_index=validator_index, - genesis_validators_root=NETWORKS[network].GENESIS_VALIDATORS_ROOT, + genesis_validators_root=settings.network_config.GENESIS_VALIDATORS_ROOT, fork=fork, ) diff --git a/src/validators/keystores/remote.py b/src/validators/keystores/remote.py index bdb63633..fbf1bae1 100644 --- a/src/validators/keystores/remote.py +++ b/src/validators/keystores/remote.py @@ -9,13 +9,8 @@ from sw_utils.typings import ConsensusFork from web3 import Web3 -from src.common.typings import Oracles -from src.config.networks import NETWORKS from src.config.settings import REMOTE_SIGNER_TIMEOUT, settings from src.validators.keystores.base import BaseKeystore -from src.validators.signing.common import encrypt_signature -from src.validators.signing.key_shares import bls_signature_and_public_key_to_shares -from src.validators.typings import ExitSignatureShards from src.validators.utils import load_deposit_data logger = logging.getLogger(__name__) @@ -70,47 +65,14 @@ def __contains__(self, public_key): def public_keys(self) -> list[HexStr]: return self._public_keys - async def get_exit_signature_shards( - self, - validator_index: int, - public_key: HexStr, - oracles: Oracles, - fork: ConsensusFork, - ) -> ExitSignatureShards: - message = get_exit_message_signing_root( - validator_index=validator_index, - genesis_validators_root=settings.network_config.GENESIS_VALIDATORS_ROOT, - fork=fork, - ) - - public_key_bytes = BLSPubkey(Web3.to_bytes(hexstr=public_key)) - threshold = oracles.exit_signature_recover_threshold - total = len(oracles.public_keys) - - exit_signature = await self._sign(public_key_bytes, validator_index, fork, message) - - exit_signature_shares, public_key_shares = bls_signature_and_public_key_to_shares( - message, exit_signature, public_key_bytes, threshold, total - ) - - encrypted_exit_signature_shares: list[HexStr] = [] - - for exit_signature_share, oracle_pubkey in zip(exit_signature_shares, oracles.public_keys): - encrypted_exit_signature_shares.append( - encrypt_signature(oracle_pubkey, exit_signature_share) - ) - - return ExitSignatureShards( - public_keys=[Web3.to_hex(p) for p in public_key_shares], - exit_signatures=encrypted_exit_signature_shares, - ) - async def get_exit_signature( - self, validator_index: int, public_key: HexStr, network: str, fork: ConsensusFork + self, validator_index: int, public_key: HexStr, fork: ConsensusFork | None = None ) -> BLSSignature: + fork = fork or settings.network_config.SHAPELLA_FORK + message = get_exit_message_signing_root( validator_index=validator_index, - genesis_validators_root=NETWORKS[network].GENESIS_VALIDATORS_ROOT, + genesis_validators_root=settings.network_config.GENESIS_VALIDATORS_ROOT, fork=fork, ) public_key_bytes = BLSPubkey(Web3.to_bytes(hexstr=public_key)) diff --git a/src/validators/keystores/tests/__init__.py b/src/validators/keystores/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/validators/keystores/tests/test_hashi_vault.py b/src/validators/keystores/tests/test_hashi_vault.py new file mode 100644 index 00000000..5e724ffe --- /dev/null +++ b/src/validators/keystores/tests/test_hashi_vault.py @@ -0,0 +1,51 @@ +import pytest + +from src.config.settings import settings +from src.validators.keystores.hashi_vault import ( + HashiVaultConfiguration, + HashiVaultKeystore, +) + + +class TestHashiVault: + @pytest.mark.usefixtures('mocked_hashi_vault') + async def test_hashi_vault_keystores_loading( + self, + hashi_vault_url: str, + ): + settings.hashi_vault_url = hashi_vault_url + settings.hashi_vault_token = 'Secret' + settings.hashi_vault_key_path = 'ethereum/signing/keystores' + + config = HashiVaultConfiguration.from_settings() + + keystore = await HashiVaultKeystore._load_hashi_vault_keys(config) + + assert len(keystore) == 2 + + @pytest.mark.usefixtures('mocked_hashi_vault') + async def test_hashi_vault_keystores_not_configured( + self, + hashi_vault_url: str, + ): + settings.hashi_vault_url = hashi_vault_url + settings.hashi_vault_token = None + settings.hashi_vault_key_path = None + + with pytest.raises(RuntimeError, match='URL, token and key path must be specified'): + await HashiVaultConfiguration.from_settings() + + @pytest.mark.usefixtures('mocked_hashi_vault') + async def test_hashi_vault_keystores_inaccessible( + self, + hashi_vault_url: str, + ): + settings.hashi_vault_url = hashi_vault_url + settings.hashi_vault_token = 'Secret' + settings.hashi_vault_key_path = 'ethereum/inaccessible/keystores' + + with pytest.raises( + RuntimeError, match='Can not retrieve validator signing keys from hashi vault' + ): + config = HashiVaultConfiguration.from_settings() + await HashiVaultKeystore._load_hashi_vault_keys(config) diff --git a/src/validators/signing/common.py b/src/validators/signing/common.py index 167ebda7..fbbe7b72 100644 --- a/src/validators/signing/common.py +++ b/src/validators/signing/common.py @@ -1,19 +1,35 @@ import ecies -from eth_typing import BLSSignature, HexStr +from eth_typing import BLSPubkey, BLSSignature, HexStr from multiproof import StandardMerkleTree from multiproof.standard import MultiProof -from sw_utils import get_eth1_withdrawal_credentials +from sw_utils import ( + ConsensusFork, + get_eth1_withdrawal_credentials, + get_exit_message_signing_root, +) from sw_utils.signing import compute_deposit_data from web3 import Web3 +from src.common.typings import Oracles from src.config.settings import DEPOSIT_AMOUNT_GWEI, settings -from src.validators.typings import Validator +from src.validators.keystores.base import BaseKeystore +from src.validators.signing.key_shares import bls_signature_and_public_key_to_shares +from src.validators.typings import ExitSignatureShards, Validator def encrypt_signature(oracle_pubkey: HexStr, signature: BLSSignature) -> HexStr: return Web3.to_hex(ecies.encrypt(oracle_pubkey, signature)) +def encrypt_signatures_list( + oracle_pubkeys: list[HexStr], signatures: list[BLSSignature] +) -> list[HexStr]: + res: list[HexStr] = [] + for signature, oracle_pubkey in zip(signatures, oracle_pubkeys): + res.append(encrypt_signature(oracle_pubkey, signature)) + return res + + def get_validators_proof( tree: StandardMerkleTree, validators: list[Validator], @@ -40,3 +56,51 @@ def encode_tx_validator(withdrawal_credentials: bytes, validator: Validator) -> signature=signature, ).hash_tree_root return public_key + signature + deposit_root + + +# pylint: disable-next=too-many-arguments +async def get_encrypted_exit_signature_shards( + keystore: BaseKeystore | None, + public_key: HexStr, + validator_index: int, + oracles: Oracles, + exit_signature: BLSSignature | None = None, + fork: ConsensusFork | None = None, +) -> ExitSignatureShards: + """ + * generates exit signature shards, + * generates public key shards + * encrypts exit signature shards with oracles' public keys. + """ + fork = fork or settings.network_config.SHAPELLA_FORK + + message = get_exit_message_signing_root( + validator_index=validator_index, + genesis_validators_root=settings.network_config.GENESIS_VALIDATORS_ROOT, + fork=fork, + ) + + if exit_signature is None: + if keystore is None: + raise RuntimeError('keystore or exit_signature must be set') + + exit_signature = await keystore.get_exit_signature( + validator_index=validator_index, + public_key=public_key, + fork=fork, + ) + public_key_bytes = BLSPubkey(Web3.to_bytes(hexstr=public_key)) + threshold = oracles.exit_signature_recover_threshold + total = len(oracles.public_keys) + + exit_signature_shares, public_key_shares = bls_signature_and_public_key_to_shares( + message, exit_signature, public_key_bytes, threshold, total + ) + + encrypted_exit_signature_shards = encrypt_signatures_list( + oracles.public_keys, exit_signature_shares + ) + return ExitSignatureShards( + public_keys=[Web3.to_hex(p) for p in public_key_shares], + exit_signatures=encrypted_exit_signature_shards, + ) diff --git a/src/validators/signing/tests/test_signing.py b/src/validators/signing/tests/test_common.py similarity index 64% rename from src/validators/signing/tests/test_signing.py rename to src/validators/signing/tests/test_common.py index 57c24cbb..d6aa3b49 100644 --- a/src/validators/signing/tests/test_signing.py +++ b/src/validators/signing/tests/test_common.py @@ -1,25 +1,24 @@ import random from typing import Callable +import milagro_bls_binding as bls import pytest from eth_typing import BLSSignature from eth_typing.bls import BLSPubkey +from sw_utils import get_exit_message_signing_root from sw_utils.typings import ConsensusFork from web3 import Web3 from src.common.typings import Oracles from src.config.settings import settings -from src.validators.keystores.hashi_vault import ( - HashiVaultConfiguration, - HashiVaultKeystore, -) from src.validators.keystores.local import LocalKeystore from src.validators.keystores.remote import RemoteSignerKeystore +from src.validators.signing.common import get_encrypted_exit_signature_shards from src.validators.signing.tests.oracle_functions import OracleCommittee from src.validators.typings import ExitSignatureShards -class TestSigning: +class TestGetEncryptedExitSignatureShards: @staticmethod def check_signature_shards( shards: ExitSignatureShards, @@ -60,7 +59,7 @@ def check_signature_shards( ], indirect=True, ) - async def test_get_exit_signature_shards_local( + async def test_local( self, create_validator_keypair: Callable, fork: ConsensusFork, @@ -70,16 +69,16 @@ async def test_get_exit_signature_shards_local( validator_privkey, validator_pubkey = create_validator_keypair() validator_index = 123 - shards = await LocalKeystore( - {validator_pubkey: validator_privkey} - ).get_exit_signature_shards( + keystore = LocalKeystore({validator_pubkey: validator_privkey}) + shards = await get_encrypted_exit_signature_shards( + keystore=keystore, validator_index=validator_index, public_key=validator_pubkey, oracles=mocked_oracles, fork=fork, ) - TestSigning.check_signature_shards( + self.check_signature_shards( shards=shards, committee=_mocked_oracle_committee, validator_pubkey=BLSPubkey(Web3.to_bytes(hexstr=validator_pubkey)), @@ -96,7 +95,7 @@ async def test_get_exit_signature_shards_local( ], indirect=True, ) - async def test_get_exit_signature_shards_remote_signer( + async def test_remote_signer( self, create_validator_keypair: Callable, fork: ConsensusFork, @@ -109,18 +108,66 @@ async def test_get_exit_signature_shards_remote_signer( validator_pubkey = keystore.public_keys[0] validator_index = random.randint(1, 10000) - exit_signature = await keystore.get_exit_signature( - validator_index, validator_pubkey, settings.network, fork + exit_signature = await keystore.get_exit_signature(validator_index, validator_pubkey, fork) + + shards = await get_encrypted_exit_signature_shards( + keystore=keystore, + validator_index=validator_index, + public_key=validator_pubkey, + oracles=mocked_oracles, + fork=fork, ) - shards = await keystore.get_exit_signature_shards( + self.check_signature_shards( + shards=shards, + committee=_mocked_oracle_committee, + validator_pubkey=BLSPubkey(Web3.to_bytes(hexstr=validator_pubkey)), + validator_index=validator_index, + fork=fork, + exit_signature=exit_signature, + ) + + @pytest.mark.usefixtures('fake_settings') + @pytest.mark.parametrize( + ['_mocked_oracle_committee'], + [ + pytest.param((1, 1), id='Single oracle'), + pytest.param((10, 5), id='10 oracles with recovery threshold of 5'), + pytest.param((10, 10), id='10 oracles with recovery threshold of 10'), + ], + indirect=True, + ) + async def test_api( + self, + create_validator_keypair: Callable, + fork: ConsensusFork, + mocked_oracles: Oracles, + _mocked_oracle_committee: OracleCommittee, + ): + """ + The case when settings.validators_registration_mode == ValidatorsRegistrationMode.API. + Exit signature is created by third party. + """ + validator_privkey, validator_pubkey = create_validator_keypair() + validator_index = 123 + + message = get_exit_message_signing_root( + validator_index=validator_index, + genesis_validators_root=settings.network_config.GENESIS_VALIDATORS_ROOT, + fork=fork, + ) + exit_signature = bls.Sign(validator_privkey, message) + + shards = await get_encrypted_exit_signature_shards( + keystore=None, validator_index=validator_index, public_key=validator_pubkey, oracles=mocked_oracles, fork=fork, + exit_signature=exit_signature, ) - TestSigning.check_signature_shards( + self.check_signature_shards( shards=shards, committee=_mocked_oracle_committee, validator_pubkey=BLSPubkey(Web3.to_bytes(hexstr=validator_pubkey)), @@ -141,55 +188,13 @@ async def test_remote_signer_pubkey_not_present( _, bls_pubkey = create_validator_keypair() validator_index = 123 settings.remote_signer_url = remote_signer_url - ''' - [BLSPubkey(Web3.to_bytes(hexstr=bls_pubkey))] - ''' + keystore = RemoteSignerKeystore([]) + with pytest.raises(RuntimeError, match='Failed to get signature'): - _ = await RemoteSignerKeystore({}).get_exit_signature_shards( + _ = await get_encrypted_exit_signature_shards( + keystore=keystore, validator_index=validator_index, public_key=bls_pubkey, oracles=mocked_oracles, fork=fork, ) - - @pytest.mark.usefixtures('mocked_hashi_vault') - async def test_hashi_vault_keystores_loading( - self, - hashi_vault_url: str, - ): - settings.hashi_vault_url = hashi_vault_url - settings.hashi_vault_token = 'Secret' - settings.hashi_vault_key_path = 'ethereum/signing/keystores' - - config = HashiVaultConfiguration.from_settings() - - keystore = await HashiVaultKeystore._load_hashi_vault_keys(config) - - assert len(keystore) == 2 - - @pytest.mark.usefixtures('mocked_hashi_vault') - async def test_hashi_vault_keystores_not_configured( - self, - hashi_vault_url: str, - ): - settings.hashi_vault_url = hashi_vault_url - settings.hashi_vault_token = None - settings.hashi_vault_key_path = None - - with pytest.raises(RuntimeError, match='URL, token and key path must be specified'): - await HashiVaultConfiguration.from_settings() - - @pytest.mark.usefixtures('mocked_hashi_vault') - async def test_hashi_vault_keystores_inaccessible( - self, - hashi_vault_url: str, - ): - settings.hashi_vault_url = hashi_vault_url - settings.hashi_vault_token = 'Secret' - settings.hashi_vault_key_path = 'ethereum/inaccessible/keystores' - - with pytest.raises( - RuntimeError, match='Can not retrieve validator signing keys from hashi vault' - ): - config = HashiVaultConfiguration.from_settings() - await HashiVaultKeystore._load_hashi_vault_keys(config) diff --git a/src/validators/tasks.py b/src/validators/tasks.py index fb1747b3..b324c25d 100644 --- a/src/validators/tasks.py +++ b/src/validators/tasks.py @@ -2,6 +2,7 @@ import logging import time +from eth_typing import HexStr from multiproof.standard import MultiProof from sw_utils import EventScanner, IpfsFetchClient from sw_utils.typings import Bytes32 @@ -15,7 +16,7 @@ from src.common.metrics import metrics from src.common.tasks import BaseTask from src.common.typings import Oracles -from src.common.utils import MGNO_RATE, WAD, get_current_timestamp +from src.common.utils import MGNO_RATE, WAD, get_current_timestamp, log_verbose from src.config.networks import GNOSIS from src.config.settings import DEPOSIT_AMOUNT, settings from src.validators.database import NetworkValidatorCrud @@ -29,22 +30,29 @@ update_unused_validator_keys_metric, ) from src.validators.keystores.base import BaseKeystore -from src.validators.signing.common import get_validators_proof +from src.validators.signing.common import ( + get_encrypted_exit_signature_shards, + get_validators_proof, +) from src.validators.typings import ( ApprovalRequest, DepositData, NetworkValidator, Validator, + ValidatorsRegistrationMode, ) from src.validators.utils import send_approval_requests logger = logging.getLogger(__name__) +pending_validator_registrations: list[HexStr] = [] + + class ValidatorsTask(BaseTask): def __init__( self, - keystore: BaseKeystore, + keystore: BaseKeystore | None, deposit_data: DepositData, ): self.keystore = keystore @@ -57,22 +65,45 @@ async def process_block(self) -> None: # process new network validators await self.network_validators_scanner.process_new_events(chain_state.execution_block) - # check and register new validators + + if self.keystore is None: + return + await update_unused_validator_keys_metric( keystore=self.keystore, deposit_data=self.deposit_data, ) - await register_validators( - keystore=self.keystore, - deposit_data=self.deposit_data, + if settings.validators_registration_mode == ValidatorsRegistrationMode.AUTO: + # check and register new validators + await register_validators( + keystore=self.keystore, + deposit_data=self.deposit_data, + ) + + +async def register_and_remove_pending_validators( + keystore: BaseKeystore | None, + deposit_data: DepositData, + validators: list[Validator], +) -> HexStr | None: + try: + return await register_validators( + keystore=keystore, deposit_data=deposit_data, validators=validators ) + except Exception as e: + log_verbose(e) + return None + finally: + for validator in validators: + pending_validator_registrations.remove(validator.public_key) -# pylint: disable-next=too-many-locals,too-many-branches +# pylint: disable-next=too-many-locals,too-many-branches,too-many-return-statements,too-many-statements async def register_validators( - keystore: BaseKeystore, + keystore: BaseKeystore | None, deposit_data: DepositData, -) -> None: + validators: list[Validator] | None = None, +) -> HexStr | None: """Registers vault validators.""" if ( settings.network_config.IS_SUPPORT_V2_MIGRATION @@ -82,41 +113,28 @@ async def register_validators( logger.info( 'Waiting for vault to become owner of v2 pool escrow to start registering validators...' ) - return - - vault_balance, update_state_call = await get_withdrawable_assets() - if settings.network == GNOSIS: - # apply GNO -> mGNO exchange rate - vault_balance = Wei(int(vault_balance * MGNO_RATE // WAD)) - - metrics.stakeable_assets.set(int(vault_balance)) + return None - # calculate number of validators that can be registered - validators_count = vault_balance // DEPOSIT_AMOUNT - if not validators_count: - # not enough balance to register validators - return - - # get latest oracles - oracles = await get_oracles() + _, update_state_call = await get_withdrawable_assets() - validators_count = min(oracles.validators_approval_batch_limit, validators_count) + if validators is None and keystore is None: + raise RuntimeError('validators or keystore must be set') - if not await check_gas_price(): - return + if validators is None: + validators = await get_available_validators_for_registration( + keystore=keystore, deposit_data=deposit_data + ) - validators: list[Validator] = await get_available_validators( - keystore=keystore, - deposit_data=deposit_data, - count=validators_count, - ) if not validators: logger.warning( 'There are no available validators in the current deposit data ' 'to proceed with registration. ' 'To register additional validators, you must upload new deposit data.' ) - return + return None + + if not await check_gas_price(): + return None logger.info('Started registration of %d validator(s)', len(validators)) @@ -126,6 +144,7 @@ async def register_validators( ) registry_root = None oracles_request = None + oracles = await get_oracles() deadline = get_current_timestamp() + oracles.signature_validity_period approvals_min_interval = 1 @@ -169,7 +188,9 @@ async def register_validators( logger.info( 'Registry root has changed during validators registration. Retrying...', ) - return + return None + + tx_hash: HexStr | None = None if len(validators) == 1: validator = validators[0] @@ -184,8 +205,7 @@ async def register_validators( logger.info( 'Successfully registered validator with public key %s', validator.public_key ) - - if len(validators) > 1: + elif len(validators) > 1: tx_hash = await register_multiple_validator( approval=oracles_approval, multi_proof=multi_proof, @@ -197,11 +217,51 @@ async def register_validators( pub_keys = ', '.join([val.public_key for val in validators]) logger.info('Successfully registered validators with public keys %s', pub_keys) + return tx_hash + + +async def get_available_validators_for_registration( + keystore: BaseKeystore | None, + deposit_data: DepositData, + run_check_deposit_data_root: bool = True, +) -> list[Validator]: + validators_count = await get_validators_count_from_vault_assets() + + if not validators_count: + # not enough balance to register validators + return [] + + # get latest oracles + oracles = await get_oracles() + + validators_count = min(oracles.validators_approval_batch_limit, validators_count) + + validators = await get_available_validators( + keystore=keystore, + deposit_data=deposit_data, + count=validators_count, + run_check_deposit_data_root=run_check_deposit_data_root, + ) + return validators + + +async def get_validators_count_from_vault_assets() -> int: + vault_balance, _ = await get_withdrawable_assets() + if settings.network == GNOSIS: + # apply GNO -> mGNO exchange rate + vault_balance = Wei(int(vault_balance * MGNO_RATE // WAD)) + + metrics.stakeable_assets.set(int(vault_balance)) + + # calculate number of validators that can be registered + validators_count = vault_balance // DEPOSIT_AMOUNT + return validators_count + # pylint: disable-next=too-many-arguments async def create_approval_request( oracles: Oracles, - keystore: BaseKeystore, + keystore: BaseKeystore | None, validators: list[Validator], registry_root: Bytes32, multi_proof: MultiProof, @@ -211,12 +271,14 @@ async def create_approval_request( # get next validator index for exit signature latest_public_keys = await get_latest_network_validator_public_keys() - validator_index = NetworkValidatorCrud().get_next_validator_index(list(latest_public_keys)) - logger.debug('Next validator index for exit signature: %d', validator_index) + start_validator_index = NetworkValidatorCrud().get_next_validator_index( + list(latest_public_keys) + ) + logger.debug('Next validator index for exit signature: %d', start_validator_index) # get exit signature shards request = ApprovalRequest( - validator_index=validator_index, + validator_index=start_validator_index, vault_address=settings.vault, validators_root=Web3.to_hex(registry_root), public_keys=[], @@ -228,12 +290,13 @@ async def create_approval_request( proof_indexes=[val[1] for val in multi_proof.leaves], deadline=deadline, ) - for validator in validators: - shards = await keystore.get_exit_signature_shards( - validator_index=validator_index, + for validator_index, validator in enumerate(validators, start_validator_index): + shards = await get_encrypted_exit_signature_shards( + keystore=keystore, public_key=validator.public_key, + validator_index=validator_index, oracles=oracles, - fork=settings.network_config.SHAPELLA_FORK, + exit_signature=validator.exit_signature, ) if not shards: @@ -247,7 +310,6 @@ async def create_approval_request( request.public_key_shards.append(shards.public_keys) request.exit_signature_shards.append(shards.exit_signatures) - validator_index += 1 return request diff --git a/src/validators/typings.py b/src/validators/typings.py index 2e56b655..4c2b8b5c 100644 --- a/src/validators/typings.py +++ b/src/validators/typings.py @@ -1,7 +1,8 @@ from dataclasses import dataclass +from enum import Enum from typing import NewType -from eth_typing import BlockNumber, ChecksumAddress, HexStr +from eth_typing import BlockNumber, BLSSignature, ChecksumAddress, HexStr from multiproof import StandardMerkleTree from sw_utils.typings import Bytes32 @@ -19,6 +20,7 @@ class Validator: deposit_data_index: int public_key: HexStr signature: HexStr + exit_signature: BLSSignature | None = None @dataclass @@ -59,3 +61,13 @@ class KeeperApprovalParams: validators: HexStr | bytes signatures: HexStr | bytes exitSignaturesIpfsHash: str + + +class ValidatorsRegistrationMode(Enum): + """ + AUTO mode: validators are registered automatically when vault assets are enough. + API mode: validators registration is triggered by API request. + """ + + AUTO = 'AUTO' + API = 'API'