Merge stage into master (#82)

* Add tests for controllers

Signed-off-by: cyc60 <[email protected]>

* Move pagination to gql query funcs

Signed-off-by: cyc60 <[email protected]>

* Fix a nested bug in tests for clients

Signed-off-by: cyc60 <[email protected]>

* Implement batch deposits (#79)

* Implement batch deposits

* Update packages to latest

* Fix type

* Fix tests

* Fix validator selection

* Updaate pyproject version

* Add faker for test data

Signed-off-by: cyc60 <[email protected]>

* Increase batch param (#81)

Co-authored-by: cyc60 <[email protected]>

Author: tsudmi

.github/workflows/ci.yaml

@@ -20,6 +20,38 @@ jobs:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v2
     - uses: pre-commit/[email protected]
+  test:
+    name: Testing
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8.12
+
+      # Install poetry
+      - name: Load cached Poetry installation
+        uses: actions/cache@v2
+        with:
+          path: ~/.local
+          key: poetry-0
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      # Install dependencies
+      - name: Install dependencies
+        run: poetry install --no-interaction --no-root
+
+      # Run tests
+      - name: Run tests
+        run: poetry run pytest -s oracle/
   docker:
     name: Build Docker Image
     runs-on: ubuntu-latest

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/psf/black
-    rev: 22.1.0
+    rev: 22.3.0
     hooks:
       - id: black
 

oracle/keeper/contracts.py

@@ -167,14 +167,14 @@ def get_oracles_contract(web3_client: Web3, network: str) -> Contract:
                                 "type": "bytes",
                             },
                         ],
-                        "internalType": "struct IPoolValidators.DepositData",
+                        "internalType": "struct IPoolValidators.DepositData[]",
                         "name": "depositData",
-                        "type": "tuple",
+                        "type": "tuple[]",
                     },
                     {
-                        "internalType": "bytes32[]",
-                        "name": "merkleProof",
-                        "type": "bytes32[]",
+                        "internalType": "bytes32[][]",
+                        "name": "merkleProofs",
+                        "type": "bytes32[][]",
                     },
                     {
                         "internalType": "bytes32",
@@ -187,7 +187,7 @@ def get_oracles_contract(web3_client: Web3, network: str) -> Contract:
                         "type": "bytes[]",
                     },
                 ],
-                "name": "registerValidator",
+                "name": "registerValidators",
                 "outputs": [],
                 "stateMutability": "nonpayable",
                 "type": "function",

oracle/keeper/typings.py

@@ -4,7 +4,7 @@
 
 from oracle.oracle.distributor.types import DistributorVote
 from oracle.oracle.rewards.types import RewardVote
-from oracle.oracle.validators.types import ValidatorVote
+from oracle.oracle.validators.types import ValidatorsVote
 
 
 class Parameters(NamedTuple):
@@ -17,4 +17,4 @@ class Parameters(NamedTuple):
 class OraclesVotes(NamedTuple):
     rewards: List[RewardVote]
     distributor: List[DistributorVote]
-    validator: List[ValidatorVote]
+    validators: List[ValidatorsVote]

oracle/keeper/utils.py

@@ -1,3 +1,4 @@
+import json
 import logging
 import time
 from collections import Counter
@@ -16,7 +17,7 @@
 from oracle.networks import NETWORKS
 from oracle.oracle.distributor.types import DistributorVote
 from oracle.oracle.rewards.types import RewardVote
-from oracle.oracle.validators.types import ValidatorVote
+from oracle.oracle.validators.types import ValidatorsVote
 from oracle.settings import (
     CONFIRMATION_BLOCKS,
     DISTRIBUTOR_VOTE_FILENAME,
@@ -140,16 +141,26 @@ def check_distributor_vote(
 
 
 def check_validator_vote(
-    web3_client: Web3, vote: ValidatorVote, oracle: ChecksumAddress
+    web3_client: Web3, vote: ValidatorsVote, oracle: ChecksumAddress
 ) -> bool:
     """Checks whether oracle's validator vote is correct."""
     try:
+        deposit_data_payloads = []
+        for deposit_data in vote["deposit_data"]:
+            deposit_data_payloads.append(
+                (
+                    deposit_data["operator"],
+                    deposit_data["withdrawal_credentials"],
+                    deposit_data["deposit_data_root"],
+                    deposit_data["public_key"],
+                    deposit_data["deposit_data_signature"],
+                )
+            )
         encoded_data: bytes = web3_client.codec.encode_abi(
-            ["uint256", "bytes", "address", "bytes32"],
+            ["uint256", "(address,bytes32,bytes32,bytes,bytes)[]", "bytes32"],
             [
                 int(vote["nonce"]),
-                vote["public_key"],
-                vote["operator"],
+                deposit_data_payloads,
                 vote["validators_deposit_root"],
             ],
         )
@@ -168,7 +179,7 @@ def get_oracles_votes(
     oracles: List[ChecksumAddress],
 ) -> OraclesVotes:
     """Fetches oracle votes that match current nonces."""
-    votes = OraclesVotes(rewards=[], distributor=[], validator=[])
+    votes = OraclesVotes(rewards=[], distributor=[], validators=[])
     network_config = NETWORKS[network]
     aws_bucket_name = network_config["AWS_BUCKET_NAME"]
     aws_region = network_config["AWS_REGION"]
@@ -183,7 +194,7 @@ def get_oracles_votes(
                 check_distributor_vote,
             ),
             (
-                votes.validator,
+                votes.validators,
                 VALIDATOR_VOTE_FILENAME,
                 validators_nonce,
                 check_validator_vote,
@@ -349,57 +360,67 @@ def submit_votes(
 
     counter = Counter(
         [
-            (vote["public_key"], vote["operator"], vote["validators_deposit_root"])
-            for vote in votes.validator
+            (
+                json.dumps(vote["deposit_data"], sort_keys=True),
+                vote["validators_deposit_root"],
+            )
+            for vote in votes.validators
         ]
     )
     most_voted = counter.most_common(1)
     if most_voted and can_submit(most_voted[0][1], total_oracles):
-        public_key, operator, validators_deposit_root = most_voted[0][0]
+        deposit_data, validators_deposit_root = most_voted[0][0]
+        deposit_data = json.loads(deposit_data)
+
         signatures = []
         i = 0
         while not can_submit(len(signatures), total_oracles):
-            vote = votes.validator[i]
-            if (public_key, operator, validators_deposit_root) == (
-                vote["public_key"],
-                vote["operator"],
+            vote = votes.validators[i]
+            if (deposit_data, validators_deposit_root) == (
+                vote["deposit_data"],
                 vote["validators_deposit_root"],
             ):
                 signatures.append(vote["signature"])
             i += 1
 
-        validator_vote: ValidatorVote = next(
+        validators_vote: ValidatorsVote = next(
             vote
-            for vote in votes.validator
-            if (public_key, operator, validators_deposit_root)
+            for vote in votes.validators
+            if (deposit_data, validators_deposit_root)
             == (
-                vote["public_key"],
-                vote["operator"],
+                vote["deposit_data"],
                 vote["validators_deposit_root"],
             )
         )
         logger.info(
-            f"[{network}] Submitting validator registration: "
-            f"operator={operator}, "
-            f"public key={public_key}, "
-            f"validator deposit root={validators_deposit_root}"
+            f"[{network}] Submitting validator(s) registration: "
+            f"count={len(validators_vote['deposit_data'])}, "
+            f"deposit root={validators_deposit_root}"
         )
+        submit_deposit_data = []
+        submit_merkle_proofs = []
+        for deposit in deposit_data:
+            submit_deposit_data.append(
+                dict(
+                    operator=deposit["operator"],
+                    withdrawalCredentials=deposit["withdrawal_credentials"],
+                    depositDataRoot=deposit["deposit_data_root"],
+                    publicKey=deposit["public_key"],
+                    signature=deposit["deposit_data_signature"],
+                )
+            )
+            submit_merkle_proofs.append(deposit["proof"])
+
         submit_update(
             network,
             web3_client,
-            oracles_contract.functions.registerValidator(
-                dict(
-                    operator=validator_vote["operator"],
-                    withdrawalCredentials=validator_vote["withdrawal_credentials"],
-                    depositDataRoot=validator_vote["deposit_data_root"],
-                    publicKey=validator_vote["public_key"],
-                    signature=validator_vote["deposit_data_signature"],
-                ),
-                validator_vote["proof"],
+            oracles_contract.functions.registerValidators(
+                submit_deposit_data,
+                submit_merkle_proofs,
                 validators_deposit_root,
                 signatures,
             ),
         )
         logger.info(
-            f"[{network}] Validator registration has been successfully submitted"
+            f"[{network}] Validator(s) registration has been successfully submitted"
         )

oracle/networks.py

@@ -29,6 +29,16 @@
             default="https://api.thegraph.com/subgraphs/name/stakewise/uniswap-v3-mainnet",
         ),
         ETH2_ENDPOINT=config(f"{MAINNET_UPPER}_ETH2_ENDPOINT", default=""),
+        VALIDATORS_FETCH_CHUNK_SIZE=config(
+            f"{MAINNET_UPPER}_VALIDATORS_FETCH_CHUNK_SIZE",
+            default=100,
+            cast=int,
+        ),
+        VALIDATORS_BATCH_SIZE=config(
+            f"{MAINNET_UPPER}_VALIDATORS_BATCH_SIZE",
+            default=10,
+            cast=int,
+        ),
         SLOTS_PER_EPOCH=32,
         SECONDS_PER_SLOT=12,
         ORACLES_CONTRACT_ADDRESS=Web3.toChecksumAddress(
@@ -92,6 +102,16 @@
             default="https://api.thegraph.com/subgraphs/name/stakewise/uniswap-v3-goerli",
         ),
         ETH2_ENDPOINT=config(f"{GOERLI_UPPER}_ETH2_ENDPOINT", default=""),
+        VALIDATORS_FETCH_CHUNK_SIZE=config(
+            f"{GOERLI_UPPER}_VALIDATORS_FETCH_CHUNK_SIZE",
+            default=100,
+            cast=int,
+        ),
+        VALIDATORS_BATCH_SIZE=config(
+            f"{GOERLI_UPPER}_VALIDATORS_BATCH_SIZE",
+            default=10,
+            cast=int,
+        ),
         SLOTS_PER_EPOCH=32,
         SECONDS_PER_SLOT=12,
         ORACLES_CONTRACT_ADDRESS=Web3.toChecksumAddress(
@@ -153,6 +173,16 @@
             default="",
         ),
         ETH2_ENDPOINT=config(f"{PERM_GOERLI_UPPER}_ETH2_ENDPOINT", default=""),
+        VALIDATORS_FETCH_CHUNK_SIZE=config(
+            f"{PERM_GOERLI_UPPER}_VALIDATORS_FETCH_CHUNK_SIZE",
+            default=100,
+            cast=int,
+        ),
+        VALIDATORS_BATCH_SIZE=config(
+            f"{PERM_GOERLI_UPPER}_VALIDATORS_BATCH_SIZE",
+            default=10,
+            cast=int,
+        ),
         SLOTS_PER_EPOCH=32,
         SECONDS_PER_SLOT=12,
         ORACLES_CONTRACT_ADDRESS=Web3.toChecksumAddress(
@@ -217,6 +247,16 @@
             f"{GNOSIS_CHAIN_UPPER}_UNISWAP_V3_SUBGRAPH_URL", default=""
         ),
         ETH2_ENDPOINT=config(f"{GNOSIS_CHAIN_UPPER}_ETH2_ENDPOINT", default=""),
+        VALIDATORS_FETCH_CHUNK_SIZE=config(
+            f"{GNOSIS_CHAIN_UPPER}_VALIDATORS_FETCH_CHUNK_SIZE",
+            default=100,
+            cast=int,
+        ),
+        VALIDATORS_BATCH_SIZE=config(
+            f"{GNOSIS_CHAIN_UPPER}_VALIDATORS_BATCH_SIZE",
+            default=10,
+            cast=int,
+        ),
         SLOTS_PER_EPOCH=16,
         SECONDS_PER_SLOT=5,
         ORACLES_CONTRACT_ADDRESS=Web3.toChecksumAddress(