Skip to content

Commit

Permalink
Add api for obol (#280)
Browse files Browse the repository at this point in the history
* Add api for obol

* Uvicorn logging and signals

* Fix log message

* Fix lint

* Add test_exit_signature_shards_without_keystore

* Add run_check_deposit_data_root

* Fix pydantic version

* Fix naming, add docstring

* Move endpoints.py to api/

* Del skip_validator_registration_tx

* Use secrets

* Warm up oracles cache

* Del todo

* Add register_and_remove_pending_validators

* Fixes after merge

* Fix pip-audit starlette

* Add start-api command

* Review fixes

* Move signatures encryption out of keystore

* Fix grammar

* Fix tests

* Moved get_exit_signature_shards out of keystore

* Revert "Fix tests"

This reverts commit 97154d7.

* Adapt tests

* Del enable_api

* Rename tests

* Moved hashi vault tests

* Rename test_signing.py -> test_common.py

* Review fixes

* Rename approvals -> registration requests

* Fix slicing
  • Loading branch information
evgeny-stakewise authored Feb 21, 2024
1 parent 1cbf436 commit 7754244
Show file tree
Hide file tree
Showing 27 changed files with 1,135 additions and 314 deletions.
255 changes: 241 additions & 14 deletions poetry.lock

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ psycopg2 = "==2.9.9"
pyyaml = "==6.0.1"
aiohttp = "==3.9.3"
python-json-logger = "==2.0.7"
starlette = "==0.36.2"
uvicorn = "==0.27.0"
pydantic = "==2.5.3"

[tool.poetry.group.dev.dependencies]
pylint = "==3.0.1"
Expand Down
11 changes: 11 additions & 0 deletions src/api.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
from starlette.applications import Starlette
from starlette.routing import Route

from src.validators.api.endpoints import get_validators, submit_validators

app = Starlette(
routes=[
Route('/validators', get_validators, methods=['GET']),
Route('/validators', submit_validators, methods=['POST']),
]
)
90 changes: 4 additions & 86 deletions src/commands/start.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,10 @@

import click
from eth_typing import ChecksumAddress
from sw_utils import EventScanner, InterruptHandler

import src
from src.common.consensus import get_chain_finalized_head
from src.common.execution import WalletTask
from src.common.logging import LOG_LEVELS, setup_logging
from src.common.metrics import MetricsTask, metrics_server
from src.common.startup_check import startup_checks
from src.common.utils import get_build_version, log_verbose
from src.commands.start_base import start_base
from src.common.logging import LOG_LEVELS
from src.common.utils import log_verbose
from src.common.validators import validate_eth_address
from src.common.vault_config import VaultConfig
from src.config.settings import (
Expand All @@ -24,13 +19,6 @@
LOG_PLAIN,
settings,
)
from src.exits.tasks import ExitSignatureTask
from src.harvest.tasks import HarvestTask
from src.validators.database import NetworkValidatorCrud
from src.validators.execution import NetworkValidatorsProcessor
from src.validators.keystores.load import load_keystore
from src.validators.tasks import ValidatorsTask, load_genesis_validators
from src.validators.utils import load_deposit_data

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -264,76 +252,6 @@ def start(
)

try:
asyncio.run(main())
asyncio.run(start_base())
except Exception as e:
log_verbose(e)


async def main() -> None:
setup_logging()
setup_sentry()
log_start()

await startup_checks()

NetworkValidatorCrud().setup()

# load network validators from ipfs dump
await load_genesis_validators()

# load keystore
keystore = await load_keystore()

# load deposit data
deposit_data = load_deposit_data(settings.vault, settings.deposit_data_file)
logger.info('Loaded deposit data file %s', settings.deposit_data_file)
# start operator tasks

# periodically scan network validator updates
network_validators_processor = NetworkValidatorsProcessor()
network_validators_scanner = EventScanner(network_validators_processor)

logger.info('Syncing network validator events...')
chain_state = await get_chain_finalized_head()
await network_validators_scanner.process_new_events(chain_state.execution_block)

if settings.enable_metrics:
await metrics_server()

logger.info('Started operator service')
with InterruptHandler() as interrupt_handler:
tasks = [
ValidatorsTask(
keystore=keystore,
deposit_data=deposit_data,
).run(interrupt_handler),
ExitSignatureTask(
keystore=keystore,
).run(interrupt_handler),
MetricsTask().run(interrupt_handler),
WalletTask().run(interrupt_handler),
]
if settings.harvest_vault:
tasks.append(HarvestTask().run(interrupt_handler))

await asyncio.gather(*tasks)


def log_start() -> None:
build = get_build_version()
start_str = 'Starting operator service'

if build:
logger.info('%s, version %s, build %s', start_str, src.__version__, build)
else:
logger.info('%s, version %s', start_str, src.__version__)


def setup_sentry():
if settings.sentry_dsn:
# pylint: disable-next=import-outside-toplevel
import sentry_sdk

sentry_sdk.init(settings.sentry_dsn, traces_sample_rate=0.1)
sentry_sdk.set_tag('network', settings.network)
sentry_sdk.set_tag('vault', settings.vault)
227 changes: 227 additions & 0 deletions src/commands/start_api.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,227 @@
import asyncio
import logging
from pathlib import Path

import click
from eth_typing import ChecksumAddress

import src.validators.api.endpoints # noqa # pylint:disable=unused-import
from src.commands.start_base import start_base
from src.common.logging import LOG_LEVELS
from src.common.utils import log_verbose
from src.common.validators import validate_eth_address
from src.common.vault_config import VaultConfig
from src.config.settings import (
AVAILABLE_NETWORKS,
DEFAULT_API_HOST,
DEFAULT_API_PORT,
DEFAULT_MAX_FEE_PER_GAS_GWEI,
DEFAULT_METRICS_HOST,
DEFAULT_METRICS_PORT,
LOG_FORMATS,
LOG_PLAIN,
settings,
)
from src.validators.typings import ValidatorsRegistrationMode

logger = logging.getLogger(__name__)


@click.option(
'--data-dir',
default=str(Path.home() / '.stakewise'),
envvar='DATA_DIR',
help='Path where the vault data will be placed. Default is ~/.stakewise.',
type=click.Path(exists=True, file_okay=False, dir_okay=True),
)
@click.option(
'--database-dir',
type=click.Path(exists=True, file_okay=False, dir_okay=True),
envvar='DATABASE_DIR',
help='The directory where the database will be created or read from. '
'Default is ~/.stakewise/<vault>.',
)
@click.option(
'--max-fee-per-gas-gwei',
type=int,
envvar='MAX_FEE_PER_GAS_GWEI',
help=f'Maximum fee per gas limit for transactions. '
f'Default is {DEFAULT_MAX_FEE_PER_GAS_GWEI} Gwei.',
default=DEFAULT_MAX_FEE_PER_GAS_GWEI,
)
@click.option(
'--hot-wallet-password-file',
type=click.Path(exists=True, file_okay=True, dir_okay=False),
envvar='HOT_WALLET_PASSWORD_FILE',
help='Absolute path to the hot wallet password file. '
'Default is the file generated with "create-wallet" command.',
)
@click.option(
'--hot-wallet-file',
type=click.Path(exists=True, file_okay=True, dir_okay=False),
envvar='HOT_WALLET_FILE',
help='Absolute path to the hot wallet. '
'Default is the file generated with "create-wallet" command.',
)
@click.option(
'--deposit-data-file',
type=click.Path(exists=True, file_okay=True, dir_okay=False),
envvar='DEPOSIT_DATA_FILE',
help='Path to the deposit_data.json file. '
'Default is the file generated with "create-keys" command.',
)
@click.option(
'--network',
type=click.Choice(
AVAILABLE_NETWORKS,
case_sensitive=False,
),
envvar='NETWORK',
help='The network of the vault. Default is the network specified at "init" command.',
)
@click.option(
'--enable-metrics',
is_flag=True,
envvar='ENABLE_METRICS',
help='Whether to enable metrics server. Disabled by default.',
)
@click.option(
'--metrics-host',
type=str,
help=f'The prometheus metrics host. Default is {DEFAULT_METRICS_HOST}.',
envvar='METRICS_HOST',
default=DEFAULT_METRICS_HOST,
)
@click.option(
'--metrics-port',
type=int,
help=f'The prometheus metrics port. Default is {DEFAULT_METRICS_PORT}.',
envvar='METRICS_PORT',
default=DEFAULT_METRICS_PORT,
)
@click.option(
'-v',
'--verbose',
help='Enable debug mode. Default is false.',
envvar='VERBOSE',
is_flag=True,
)
@click.option(
'--harvest-vault',
is_flag=True,
envvar='HARVEST_VAULT',
help='Whether to submit vault harvest transactions. Default is false.',
)
@click.option(
'--execution-endpoints',
type=str,
envvar='EXECUTION_ENDPOINTS',
prompt='Enter comma separated list of API endpoints for execution nodes',
help='Comma separated list of API endpoints for execution nodes.',
)
@click.option(
'--consensus-endpoints',
type=str,
envvar='CONSENSUS_ENDPOINTS',
prompt='Enter comma separated list of API endpoints for consensus nodes',
help='Comma separated list of API endpoints for consensus nodes.',
)
@click.option(
'--vault',
type=ChecksumAddress,
callback=validate_eth_address,
envvar='VAULT',
prompt='Enter the vault address',
help='Address of the vault to register validators for.',
)
@click.option(
'--log-format',
type=click.Choice(
LOG_FORMATS,
case_sensitive=False,
),
default=LOG_PLAIN,
envvar='LOG_FORMAT',
help='The log record format. Can be "plain" or "json".',
)
@click.option(
'--log-level',
type=click.Choice(
LOG_LEVELS,
case_sensitive=False,
),
default='INFO',
envvar='LOG_LEVEL',
help='The log level.',
)
@click.option(
'--api-host',
type=str,
help=f'API host. Default is {DEFAULT_API_HOST}.',
envvar='API_HOST',
default=DEFAULT_API_HOST,
)
@click.option(
'--api-port',
type=int,
help=f'API port. Default is {DEFAULT_API_PORT}.',
envvar='API_PORT',
default=DEFAULT_API_PORT,
)
@click.command(help='Start operator service')
# pylint: disable-next=too-many-arguments,too-many-locals
def start_api(
vault: ChecksumAddress,
consensus_endpoints: str,
execution_endpoints: str,
harvest_vault: bool,
verbose: bool,
enable_metrics: bool,
metrics_host: str,
metrics_port: int,
data_dir: str,
log_level: str,
log_format: str,
network: str | None,
deposit_data_file: str | None,
hot_wallet_file: str | None,
hot_wallet_password_file: str | None,
max_fee_per_gas_gwei: int,
database_dir: str | None,
api_host: str,
api_port: int,
) -> None:
vault_config = VaultConfig(vault, Path(data_dir))
if network is None:
vault_config.load()
network = vault_config.network

validators_registration_mode = ValidatorsRegistrationMode.API

settings.set(
vault=vault,
vault_dir=vault_config.vault_dir,
consensus_endpoints=consensus_endpoints,
execution_endpoints=execution_endpoints,
harvest_vault=harvest_vault,
verbose=verbose,
enable_metrics=enable_metrics,
metrics_host=metrics_host,
metrics_port=metrics_port,
network=network,
deposit_data_file=deposit_data_file,
hot_wallet_file=hot_wallet_file,
hot_wallet_password_file=hot_wallet_password_file,
max_fee_per_gas_gwei=max_fee_per_gas_gwei,
database_dir=database_dir,
log_level=log_level,
log_format=log_format,
api_host=api_host,
api_port=api_port,
validators_registration_mode=validators_registration_mode,
)

try:
asyncio.run(start_base())
except Exception as e:
log_verbose(e)
Loading

0 comments on commit 7754244

Please sign in to comment.