From 42cbc81d39343dfbac656463ab3bd5a2904c05f2 Mon Sep 17 00:00:00 2001 From: Evgeny Gusarov Date: Tue, 23 Jan 2024 22:43:46 +0300 Subject: [PATCH] Add api for obol --- poetry.lock | 82 ++++++++++++++++++++++++++++++++++++- pyproject.toml | 2 + src/api.py | 11 +++++ src/commands/start.py | 44 +++++++++++++++++++- src/common/metrics.py | 5 +++ src/config/settings.py | 11 +++++ src/validators/endpoints.py | 69 +++++++++++++++++++++++++++++++ src/validators/tasks.py | 9 ++-- 8 files changed, 227 insertions(+), 6 deletions(-) create mode 100644 src/api.py create mode 100644 src/validators/endpoints.py diff --git a/poetry.lock b/poetry.lock index 96169fbb..419102ca 100644 --- a/poetry.lock +++ b/poetry.lock @@ -135,6 +135,28 @@ files = [ {file = "altgraph-0.17.4.tar.gz", hash = "sha256:1b5afbb98f6c4dcadb2e2ae6ab9fa994bbb8c1d75f4fa96d340f9437ae454406"}, ] +[[package]] +name = "anyio" +version = "4.2.0" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +optional = false +python-versions = ">=3.8" +files = [ + {file = "anyio-4.2.0-py3-none-any.whl", hash = "sha256:745843b39e829e108e518c489b31dc757de7d2131d53fac32bd8df268227bfee"}, + {file = "anyio-4.2.0.tar.gz", hash = "sha256:e1875bb4b4e2de1669f4bc7869b6d3f54231cdced71605e6e64c9be77e3be50f"}, +] + +[package.dependencies] +exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} +idna = ">=2.8" +sniffio = ">=1.1" +typing-extensions = {version = ">=4.1", markers = "python_version < \"3.11\""} + +[package.extras] +doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] +trio = ["trio (>=0.23)"] + [[package]] name = "asn1crypto" version = "1.5.1" @@ -1293,6 +1315,17 @@ gitdb = ">=4.0.1,<5" [package.extras] test = ["black", "coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest", "pytest-cov", "pytest-instafail", "pytest-subtests", "pytest-sugar"] +[[package]] +name = "h11" +version = "0.14.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = false +python-versions = ">=3.7" +files = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] + [[package]] name = "hexbytes" version = "0.3.1" @@ -2743,6 +2776,17 @@ files = [ {file = "smmap-5.0.1.tar.gz", hash = "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62"}, ] +[[package]] +name = "sniffio" +version = "1.3.0" +description = "Sniff out which async library your code is running under" +optional = false +python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"}, + {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, +] + [[package]] name = "staking_deposit" version = "2.4.0" @@ -2758,6 +2802,23 @@ url = "https://github.com/ethereum/staking-deposit-cli.git" reference = "v2.4.0" resolved_reference = "ef89710443814331aa2f592067dc4d6995cc4f6e" +[[package]] +name = "starlette" +version = "0.36.1" +description = "The little ASGI library that shines." +optional = false +python-versions = ">=3.8" +files = [ + {file = "starlette-0.36.1-py3-none-any.whl", hash = "sha256:d5b43a72f475fd1b9707f661aa66da42d59ae16c9b2a5845b4edee4309c425ee"}, + {file = "starlette-0.36.1.tar.gz", hash = "sha256:96df8541093dfd37624b5bf980802b99750db6718dd3ca341618fbbcdd6136fb"}, +] + +[package.dependencies] +anyio = ">=3.4.0,<5" + +[package.extras] +full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart", "pyyaml"] + [[package]] name = "stevedore" version = "5.1.0" @@ -2895,6 +2956,25 @@ brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] +[[package]] +name = "uvicorn" +version = "0.27.0" +description = "The lightning-fast ASGI server." +optional = false +python-versions = ">=3.8" +files = [ + {file = "uvicorn-0.27.0-py3-none-any.whl", hash = "sha256:890b00f6c537d58695d3bb1f28e23db9d9e7a17cbcc76d7457c499935f933e24"}, + {file = "uvicorn-0.27.0.tar.gz", hash = "sha256:c855578045d45625fd027367f7653d249f7c49f9361ba15cf9624186b26b8eb6"}, +] + +[package.dependencies] +click = ">=7.0" +h11 = ">=0.8" +typing-extensions = {version = ">=4.0", markers = "python_version < \"3.11\""} + +[package.extras] +standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.4)"] + [[package]] name = "varint" version = "1.0.2" @@ -3147,4 +3227,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.11" -content-hash = "281d98b7a78d263bdeeefae84287008462193a0e2bb09a74b6ef7c6724e8c592" +content-hash = "d0e4946437dd401d9c557fa401c18b230bc6889f283bf62e8ddf79863d69fb85" diff --git a/pyproject.toml b/pyproject.toml index 62343858..5392aea7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,8 @@ psycopg2 = "==2.9.9" pyyaml = "==6.0.1" aiohttp = "==3.9.1" python-json-logger = "==2.0.7" +starlette = "==0.36.1" +uvicorn = "==0.27.0" [tool.poetry.group.dev.dependencies] pylint = "==3.0.1" diff --git a/src/api.py b/src/api.py new file mode 100644 index 00000000..257d0032 --- /dev/null +++ b/src/api.py @@ -0,0 +1,11 @@ +from starlette.applications import Starlette +from starlette.routing import Route + +from src.validators.endpoints import approve_validators, get_validators + +app = Starlette( + routes=[ + Route('/validators', get_validators, methods=['GET']), + Route('/validators', approve_validators, methods=['POST']), + ] +) diff --git a/src/commands/start.py b/src/commands/start.py index 5a637ebb..91ea756d 100644 --- a/src/commands/start.py +++ b/src/commands/start.py @@ -4,10 +4,13 @@ from pathlib import Path import click +import uvicorn from eth_typing import ChecksumAddress from sw_utils import EventScanner, InterruptHandler import src +import src.validators.endpoints # noqa +from src.api import app as api_app from src.common.consensus import get_chain_finalized_head from src.common.execution import WalletTask from src.common.metrics import MetricsTask, metrics_server @@ -17,6 +20,7 @@ from src.common.vault_config import VaultConfig from src.config.settings import ( AVAILABLE_NETWORKS, + DEFAULT_API_PORT, DEFAULT_MAX_FEE_PER_GAS_GWEI, DEFAULT_METRICS_HOST, DEFAULT_METRICS_PORT, @@ -205,6 +209,26 @@ envvar='LOG_LEVEL', help='The log level.', ) +@click.option( + '--enable-api', + is_flag=True, + envvar='ENABLE_API', + help='Whether to enable API server. Disabled by default.', +) +@click.option( + '--api-port', + type=int, + help=f'API port. Default is {DEFAULT_API_PORT}.', + envvar='API_PORT', + default=DEFAULT_API_PORT, +) +@click.option( + '--enable-validators-task', + is_flag=True, + default=True, + envvar='ENABLE_VALIDATORS_TASK', + help='Whether to enable validators task. Enabled by default.', +) @click.command(help='Start operator service') # pylint: disable-next=too-many-arguments,too-many-locals def start( @@ -231,6 +255,9 @@ def start( hot_wallet_password_file: str | None, max_fee_per_gas_gwei: int, database_dir: str | None, + enable_api: bool, + api_port: int, + enable_validators_task: bool, ) -> None: vault_config = VaultConfig(vault, Path(data_dir)) if network is None: @@ -261,6 +288,9 @@ def start( database_dir=database_dir, log_level=log_level, log_format=log_format, + enable_api=enable_api, + api_port=api_port, + enable_validators_task=enable_validators_task, ) try: @@ -274,7 +304,8 @@ async def main() -> None: setup_sentry() log_start() - await startup_checks() + # todo undo + # await startup_checks() NetworkValidatorCrud().setup() @@ -297,6 +328,17 @@ async def main() -> None: chain_state = await get_chain_finalized_head() await network_validators_scanner.process_new_events(chain_state.execution_block) + if settings.enable_api: + logger.info('Starting api server') + api_app.state.keystore = keystore + api_app.state.deposit_data = deposit_data + + config = uvicorn.Config( + api_app, port=settings.api_port, log_level=settings.log_level.lower() + ) + server = uvicorn.Server(config) + asyncio.create_task(server.serve()) + if settings.enable_metrics: await metrics_server() diff --git a/src/common/metrics.py b/src/common/metrics.py index ff7fc76e..baa92a56 100644 --- a/src/common/metrics.py +++ b/src/common/metrics.py @@ -1,3 +1,5 @@ +import logging + from prometheus_client import Gauge, Info, start_http_server import src @@ -27,8 +29,11 @@ def set_app_version(self): metrics = Metrics() metrics.set_app_version() +logger = logging.getLogger(__name__) + async def metrics_server() -> None: + logger.info('Starting metrics server') start_http_server(settings.metrics_port, settings.metrics_host) diff --git a/src/config/settings.py b/src/config/settings.py index 3886c3f7..878fb8a7 100644 --- a/src/config/settings.py +++ b/src/config/settings.py @@ -12,6 +12,7 @@ DEFAULT_MAX_FEE_PER_GAS_GWEI = 100 DEFAULT_METRICS_HOST = '127.0.0.1' DEFAULT_METRICS_PORT = 9100 +DEFAULT_API_PORT = 5000 class Singleton(type): @@ -64,6 +65,10 @@ class Settings(metaclass=Singleton): sentry_dsn: str pool_size: int | None + enable_api: bool + api_port: int + enable_validators_task: bool + # pylint: disable-next=too-many-arguments,too-many-locals def set( self, @@ -91,6 +96,9 @@ def set( database_dir: str | None = None, log_level: str | None = None, log_format: str | None = None, + enable_api: bool = False, + api_port: int = DEFAULT_API_PORT, + enable_validators_task: bool = True, ) -> None: self.vault = Web3.to_checksum_address(vault) vault_dir.mkdir(parents=True, exist_ok=True) @@ -185,6 +193,9 @@ def set( self.consensus_retry_timeout = decouple_config( 'CONSENSUS_RETRY_TIMEOUT', default=120, cast=int ) + self.enable_api = enable_api + self.api_port = api_port + self.enable_validators_task = enable_validators_task @property def network_config(self) -> NetworkConfig: diff --git a/src/validators/endpoints.py b/src/validators/endpoints.py new file mode 100644 index 00000000..214842d8 --- /dev/null +++ b/src/validators/endpoints.py @@ -0,0 +1,69 @@ +import json + +from starlette.exceptions import HTTPException +from starlette.requests import Request +from starlette.responses import JSONResponse, Response +from starlette.status import HTTP_400_BAD_REQUEST +from web3.types import Wei + +from src.common.execution import get_oracles +from src.common.utils import MGNO_RATE, WAD +from src.config.networks import GNOSIS +from src.config.settings import DEPOSIT_AMOUNT, settings +from src.validators.database import NetworkValidatorCrud +from src.validators.execution import ( + get_available_validators, + get_latest_network_validator_public_keys, + get_withdrawable_assets, +) +from src.validators.typings import Validator + + +async def get_validators(request: Request) -> Response: + vault_balance, _ = await get_withdrawable_assets() + if settings.network == GNOSIS: + # apply GNO -> mGNO exchange rate + vault_balance = Wei(int(vault_balance * MGNO_RATE // WAD)) + + # calculate number of validators that can be registered + validators_count = vault_balance // DEPOSIT_AMOUNT + if not validators_count: + # not enough balance to register validators + return JSONResponse([]) + + # get latest oracles + oracles = await get_oracles() + + validators_count = min(oracles.validators_approval_batch_limit, validators_count) + + validators: list[Validator] = await get_available_validators( + keystore=request.app.state.keystore, + deposit_data=request.app.state.deposit_data, + count=validators_count, + ) + if not validators: + # All validators from `deposit_data` are already registered + return JSONResponse([]) + + # get next validator index for exit signature + latest_public_keys = await get_latest_network_validator_public_keys() + next_validator_index = NetworkValidatorCrud().get_next_validator_index(list(latest_public_keys)) + + return JSONResponse( + [ + {'public_key': validator.public_key, 'index': index} + for index, validator in enumerate(validators, next_validator_index) + ] + ) + + +def approve_validators(request: Request) -> Response: + # pylint: disable=unused-argument + try: + payload = await request.json() + except json.JSONDecodeError as exc: + raise HTTPException( + status_code=HTTP_400_BAD_REQUEST, detail='invalid_request_body' + ) from exc + + return JSONResponse([]) diff --git a/src/validators/tasks.py b/src/validators/tasks.py index 61cfd4a4..7ad1ef80 100644 --- a/src/validators/tasks.py +++ b/src/validators/tasks.py @@ -65,10 +65,11 @@ async def process_block(self) -> None: keystore=self.keystore, deposit_data=self.deposit_data, ) - await register_validators( - keystore=self.keystore, - deposit_data=self.deposit_data, - ) + # todo + # await register_validators( + # keystore=self.keystore, + # deposit_data=self.deposit_data, + # ) # pylint: disable-next=too-many-locals,too-many-branches