Duration Sampling

.github/workflows/tests.yml

@@ -49,7 +49,7 @@ jobs:
 
     - name: Upload test coverage to Codecov
       if: matrix.python-version == '3.9'
-      uses: codecov/codecov-action@v3.1.1
+      uses: codecov/codecov-action@v4.3.0
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         fail_ci_if_error: true

dev-requirements.txt

@@ -1,5 +1,5 @@
 -i https://pypi.python.org/simple
-aiohttp==3.9.3; python_version >= '3.8' and python_version < '4'
+aiohttp==3.9.4rc0; python_version >= '3.8' and python_version < '4'
 aiosignal==1.3.1; python_version >= '3.8' and python_version < '4'
 annotated-types==0.6.0
 ape-solidity==0.7.1
@@ -8,7 +8,7 @@ appnope==0.1.4
 asttokens==2.4.1
 async-timeout==4.0.3; python_version >= '3.8' and python_version < '3.11'
 attrs==23.2.0; python_version >= '3.8' and python_version < '4'
-atxm==0.2.0; python_version >= '3.8' and python_version < '4'
+atxm==0.3.0; python_version >= '3.8' and python_version < '4'
 autobahn==23.1.2; python_version >= '3.8' and python_version < '4'
 automat==22.10.0; python_version >= '3.8' and python_version < '4'
 backcall==0.2.0
@@ -38,8 +38,8 @@ eth-abi==4.2.1; python_version >= '3.8' and python_version < '4'
 eth-account==0.10.0; python_version >= '3.8' and python_version < '4'
 eth-ape==0.7.13
 eth-bloom==3.0.0
-eth-hash[pycryptodome]==0.6.0; python_version >= '3.8' and python_version < '4'
-eth-keyfile==0.7.0; python_version >= '3.8' and python_version < '4'
+eth-hash[pycryptodome]==0.7.0; python_version >= '3.8' and python_version < '4'
+eth-keyfile==0.8.0; python_version >= '3.8' and python_version < '4'
 eth-keys==0.4.0; python_version >= '3.8' and python_version < '4'
 eth-pydantic-types==0.1.0
 eth-rlp==1.0.1; python_version >= '3.8' and python_version < '4'
@@ -48,7 +48,7 @@ eth-typing==3.5.2; python_version >= '3.8' and python_version < '4'
 eth-utils==2.3.1; python_version >= '3.8' and python_version < '4'
 ethpm-types==0.6.9
 evm-trace==0.1.3
-evmchains==0.0.5
+evmchains==0.0.6
 executing==2.0.1
 filelock==3.13.4
 flask==3.0.3; python_version >= '3.8'
@@ -59,7 +59,7 @@ hexbytes==0.3.1; python_version >= '3.8' and python_version < '4'
 humanize==4.9.0; python_version >= '3.8' and python_version < '4'
 hyperlink==21.0.0; python_version >= '3.8' and python_version < '4'
 identify==2.5.35; python_version >= '3.8'
-idna==3.6; python_version >= '3.8' and python_version < '4'
+idna==3.7; python_version >= '3.8' and python_version < '4'
 ijson==3.2.3
 importlib-metadata==7.1.0; python_version >= '3.8' and python_version < '3.10'
 incremental==22.10.0; python_version >= '3.8' and python_version < '4'
@@ -72,11 +72,11 @@ jsonschema==4.21.1; python_version >= '3.8' and python_version < '4'
 jsonschema-specifications==2023.12.1; python_version >= '3.8' and python_version < '4'
 lazyasd==0.1.4
 lru-dict==1.2.0; python_version >= '3.8' and python_version < '4'
-mako==1.3.2; python_version >= '3.8' and python_version < '4'
+mako==1.3.3; python_version >= '3.8' and python_version < '4'
 markdown-it-py==3.0.0
 markupsafe==2.1.5; python_version >= '3.8' and python_version < '4'
 marshmallow==3.21.1; python_version >= '3.8' and python_version < '4'
-matplotlib-inline==0.1.6
+matplotlib-inline==0.1.7
 maya==0.6.1; python_version >= '3.8' and python_version < '4'
 mdurl==0.1.2
 mnemonic==0.20; python_version >= '3.8' and python_version < '4'
@@ -86,7 +86,7 @@ msgspec==0.18.6
 multidict==6.0.5; python_version >= '3.8' and python_version < '4'
 mypy-extensions==1.0.0
 nodeenv==1.8.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'
-nucypher@ git+https://github.com/nucypher/nucypher.git@4090a342aff2ea8a600c3694fc149bc57549483f
+nucypher@ git+https://github.com/nucypher/nucypher.git@498b9887a1b30c804bc6ca5955ddea8bea49d73a
 nucypher-core==0.13.0
 numpy==1.26.4
 packaging==23.2; python_version >= '3.8' and python_version < '4'
@@ -99,9 +99,9 @@ pickleshare==0.7.5
 platformdirs==4.2.0
 pluggy==1.4.0; python_version >= '3.8'
 pre-commit==2.21.0; python_version >= '3.7'
-prometheus-client==0.20.0; python_version >= '3.8'
+prometheus-client==0.20.0; python_version >= '3.8' and python_version < '4'
 prompt-toolkit==3.0.43
-protobuf==5.26.0rc2; python_version >= '3.8' and python_version < '4'
+protobuf==5.26.1; python_version >= '3.8' and python_version < '4'
 ptyprocess==0.7.0
 pure-eval==0.2.2
 py==1.11.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
@@ -135,19 +135,20 @@ pytest-twisted==1.14.1
 python-baseconv==1.2.2
 python-dateutil==2.9.0.post0; python_version >= '3.8' and python_version < '4'
 python-dotenv==1.0.1
+python-statemachine==2.1.2; python_version >= '3.8' and python_version < '3.13'
 pytz==2024.1; python_version >= '3.8' and python_version < '4'
 pyunormalize==15.1.0; python_version >= '3.8' and python_version < '4'
 pyyaml==6.0.1; python_version >= '3.6'
-referencing==0.33.0; python_version >= '3.8' and python_version < '4'
+referencing==0.34.0; python_version >= '3.8' and python_version < '4'
 regex==2023.12.25; python_version >= '3.8' and python_version < '4'
 requests==2.31.0; python_version >= '3.8' and python_version < '4'
 rich==13.7.1
 rlp==3.0.0; python_version >= '3.8' and python_version < '4'
-rpds-py==0.17.1; python_version >= '3.8' and python_version < '4'
+rpds-py==0.18.0; python_version >= '3.8' and python_version < '4'
 safe-pysha3==1.0.4
 semantic-version==2.10.0
 service-identity==24.1.0; python_version >= '3.8' and python_version < '4'
-setuptools==69.0.3; python_version >= '3.8' and python_version < '4'
+setuptools==69.2.0; python_version >= '3.8' and python_version < '4'
 six==1.16.0; python_version >= '3.8' and python_version < '4'
 snaptime==0.2.4; python_version >= '3.8' and python_version < '4'
 sortedcontainers==2.4.0
@@ -161,9 +162,9 @@ toolz==0.12.1; python_version >= '3.8' and python_version < '4'
 tqdm==4.66.2
 traitlets==5.14.2
 trie==2.2.0
-twisted==23.10.0; python_version >= '3.8' and python_version < '4'
+twisted==24.3.0; python_version >= '3.8' and python_version < '4'
 txaio==23.1.1; python_version >= '3.8' and python_version < '4'
-typing-extensions==4.9.0; python_version >= '3.8' and python_version < '4'
+typing-extensions==4.11.0; python_version >= '3.8' and python_version < '4'
 tzdata==2024.1; python_version >= '3.8' and python_version < '4'
 tzlocal==5.2; python_version >= '3.8' and python_version < '4'
 urllib3==2.2.0; python_version >= '3.8' and python_version < '4'
@@ -177,4 +178,4 @@ werkzeug==3.0.2; python_version >= '3.8' and python_version < '4'
 wrapt==1.16.0
 yarl==1.9.4; python_version >= '3.8' and python_version < '4'
 zipp==3.18.1; python_version >= '3.8' and python_version < '3.10'
-zope-interface==6.1; python_version >= '3.8' and python_version < '4'
+zope-interface==6.2; python_version >= '3.8' and python_version < '4'

porter/fields/base.py

@@ -53,6 +53,12 @@ def _validate(self, value):
             raise InvalidInputData(f"{self.name} must be a positive integer.")
 
 
+class NonNegativeInteger(Integer):
+    def _validate(self, value):
+        if value < 0:
+            raise InvalidInputData(f"{self.name} must be a non-negative integer.")
+
+
 class Base64BytesRepresentation(BaseField, fields.Field):
     """Serializes/Deserializes any object's byte representation to/from bae64."""
     def _serialize(self, value, attr, obj, **kwargs):

porter/interfaces.py

@@ -39,12 +39,14 @@ def get_ursulas(
         exclude_ursulas: Optional[List[ChecksumAddress]] = None,
         include_ursulas: Optional[List[ChecksumAddress]] = None,
         timeout: Optional[int] = None,
+        duration: Optional[int] = None,
     ) -> Dict:
         ursulas_info = self.implementer.get_ursulas(
             quantity=quantity,
             exclude_ursulas=exclude_ursulas,
             include_ursulas=include_ursulas,
             timeout=timeout,
+            duration=duration,
         )
 
         response_data = {"ursulas": ursulas_info}  # list of UrsulaInfo objects
@@ -101,12 +103,14 @@ def bucket_sampling(
         random_seed: Optional[int] = None,
         exclude_ursulas: Optional[List[ChecksumAddress]] = None,
         timeout: Optional[int] = None,
+        duration: Optional[int] = None,
     ) -> Dict:
         ursulas, block_number = self.implementer.bucket_sampling(
             quantity=quantity,
             random_seed=random_seed,
             exclude_ursulas=exclude_ursulas,
             timeout=timeout,
+            duration=duration,
         )
 
         response_data = {"ursulas": ursulas, "block_number": block_number}

porter/main.py

@@ -64,6 +64,9 @@ class Porter(Learner):
     DEFAULT_PORT = 9155
 
     MAX_GET_URSULAS_TIMEOUT = os.getenv("PORTER_MAX_GET_URSULAS_TIMEOUT", default=15)
+    MAX_BUCKET_SAMPLING_TIMEOUT = os.getenv(
+        "PORTER_MAX_BUCKET_SAMPLING_TIMEOUT", default=25
+    )
     MAX_DECRYPTION_TIMEOUT = os.getenv(
         "PORTER_MAX_DECRYPTION_TIMEOUT",
         default=ThresholdDecryptionClient.DEFAULT_DECRYPTION_TIMEOUT,
@@ -158,12 +161,14 @@ def get_ursulas(
         exclude_ursulas: Optional[Sequence[ChecksumAddress]] = None,
         include_ursulas: Optional[Sequence[ChecksumAddress]] = None,
         timeout: Optional[int] = None,
+        duration: Optional[int] = None,
     ) -> List[UrsulaInfo]:
         timeout = self._configure_timeout(
             "sampling", timeout, self.MAX_GET_URSULAS_TIMEOUT
         )
+        duration = duration or 0
 
-        reservoir = self._make_reservoir(exclude_ursulas, include_ursulas)
+        reservoir = self._make_reservoir(exclude_ursulas, include_ursulas, duration)
         available_nodes_to_sample = len(reservoir.values) + len(reservoir.reservoir)
         if available_nodes_to_sample < quantity:
             raise ValueError(
@@ -278,11 +283,13 @@ def _make_reservoir(
         self,
         exclude_ursulas: Optional[Sequence[ChecksumAddress]] = None,
         include_ursulas: Optional[Sequence[ChecksumAddress]] = None,
+        duration: Optional[int] = 0,
     ):
         return make_staking_provider_reservoir(
             application_agent=self.taco_child_application_agent,
             exclude_addresses=exclude_ursulas,
             include_addresses=include_ursulas,
+            duration=duration,
         )
 
     def bucket_sampling(
@@ -291,10 +298,12 @@ def bucket_sampling(
         random_seed: Optional[int] = None,
         exclude_ursulas: Optional[Sequence[ChecksumAddress]] = None,
         timeout: Optional[int] = None,
-    ) -> Tuple[List[UrsulaInfo], int]:
+        duration: Optional[int] = None,
+    ) -> Tuple[List[ChecksumAddress], int]:
         timeout = self._configure_timeout(
-            "sampling", timeout, self.MAX_GET_URSULAS_TIMEOUT
+            "bucket_sampling", timeout, self.MAX_BUCKET_SAMPLING_TIMEOUT
         )
+        duration = duration or 0
 
         if self.domain not in self._ALLOWED_DOMAINS_FOR_BUCKET_SAMPLING:
             raise ValueError("Bucket sampling is only for TACo Mainnet")
@@ -330,7 +339,9 @@ def __call__(self) -> Optional[ChecksumAddress]:
                     return None
 
         block_number = self.taco_child_application_agent.blockchain.client.block_number
-        _, sp_map = self.taco_child_application_agent.get_all_active_staking_providers()
+        _, sp_map = self.taco_child_application_agent.get_all_active_staking_providers(
+            duration=duration
+        )
         for e in exclude_ursulas or []:
             if e in sp_map:
                 del sp_map[e]
@@ -423,7 +434,7 @@ def make_sure_ursula_is_online(ursula_address) -> ChecksumAddress:
             value_factory=value_factory,
             target_successes=quantity,
             timeout=timeout,
-            stagger_timeout=10,  # TODO: Reduce it when we have a timeout for pings
+            stagger_timeout=4,  # default connection timeout for middleware calls (incl. pings) is 3s
         )
         worker_pool.start()
         try:

porter/schema.py

@@ -3,7 +3,13 @@
 from marshmallow import fields as marshmallow_fields
 
 from porter.cli.types import EIP55_CHECKSUM_ADDRESS
-from porter.fields.base import JSON, Integer, PositiveInteger, StringList
+from porter.fields.base import (
+    JSON,
+    Integer,
+    NonNegativeInteger,
+    PositiveInteger,
+    StringList,
+)
 from porter.fields.exceptions import InvalidArgumentCombo, InvalidInputData
 from porter.fields.retrieve import CapsuleFrag, RetrievalKit
 from porter.fields.taco import (
@@ -107,6 +113,18 @@ class GetUrsulas(BaseSchema):
         ),
     )
 
+    duration = NonNegativeInteger(
+        required=False,
+        load_only=True,
+        click=click.option(
+            "--duration",
+            "-d",
+            help="Minimum duration, in seconds, for which the node should be actively staking",
+            type=click.INT,
+            required=False,
+        ),
+    )
+
     # output
     ursulas = marshmallow_fields.List(marshmallow_fields.Nested(UrsulaInfoSchema), dump_only=True)
 
@@ -339,6 +357,18 @@ class BucketSampling(BaseSchema):
         ),
     )
 
+    duration = NonNegativeInteger(
+        required=False,
+        load_only=True,
+        click=click.option(
+            "--duration",
+            "-d",
+            help="Minimum duration, in seconds, for which the node should be actively staking",
+            type=click.INT,
+            required=False,
+        ),
+    )
+
     # output
     ursulas = marshmallow_fields.List(UrsulaChecksumAddress, dump_only=True)
     block_number = marshmallow_fields.Int(dump_only=True)

requirements.txt

@@ -1,10 +1,10 @@
 -i https://pypi.python.org/simple
-aiohttp==3.9.3; python_version >= '3.8' and python_version < '4'
+aiohttp==3.9.4rc0; python_version >= '3.8' and python_version < '4'
 aiosignal==1.3.1; python_version >= '3.8' and python_version < '4'
 appdirs==1.4.4; python_version >= '3.8' and python_version < '4'
 async-timeout==4.0.3; python_version < '3.11' and python_version >= '3.8'
 attrs==23.2.0; python_version >= '3.8' and python_version < '4'
-atxm==0.2.0; python_version >= '3.8' and python_version < '4'
+atxm==0.3.0; python_version >= '3.8' and python_version < '4'
 autobahn==23.1.2; python_version >= '3.8' and python_version < '4'
 automat==22.10.0; python_version >= '3.8' and python_version < '4'
 bitarray==2.9.2; python_version >= '3.8' and python_version < '4'
@@ -22,8 +22,8 @@ cytoolz==0.12.3; python_version >= '3.8' and python_version < '4' and implementa
 dateparser==1.2.0; python_version >= '3.8' and python_version < '4'
 eth-abi==4.2.1; python_version >= '3.8' and python_version < '4'
 eth-account==0.10.0; python_version >= '3.8' and python_version < '4'
-eth-hash[pycryptodome]==0.6.0; python_version >= '3.8' and python_version < '4'
-eth-keyfile==0.7.0; python_version >= '3.8' and python_version < '4'
+eth-hash[pycryptodome]==0.7.0; python_version >= '3.8' and python_version < '4'
+eth-keyfile==0.8.0; python_version >= '3.8' and python_version < '4'
 eth-keys==0.4.0; python_version >= '3.8' and python_version < '4'
 eth-rlp==1.0.1; python_version >= '3.8' and python_version < '4'
 eth-typing==3.5.2; python_version >= '3.8' and python_version < '4'
@@ -35,29 +35,29 @@ hendrix==5.0.0; python_version >= '3.8' and python_version < '4'
 hexbytes==0.3.1; python_version >= '3.8' and python_version < '4'
 humanize==4.9.0; python_version >= '3.8' and python_version < '4'
 hyperlink==21.0.0; python_version >= '3.8' and python_version < '4'
-idna==3.6; python_version >= '3.8' and python_version < '4'
+idna==3.7; python_version >= '3.8' and python_version < '4'
 importlib-metadata==7.1.0; python_version < '3.10' and python_version >= '3.8'
 incremental==22.10.0; python_version >= '3.8' and python_version < '4'
 itsdangerous==2.1.2; python_version >= '3.8' and python_version < '4'
 jinja2==3.1.3; python_version >= '3.8' and python_version < '4'
 jsonschema==4.21.1; python_version >= '3.8' and python_version < '4'
 jsonschema-specifications==2023.12.1; python_version >= '3.8' and python_version < '4'
 lru-dict==1.2.0; python_version >= '3.8' and python_version < '4'
-mako==1.3.2; python_version >= '3.8' and python_version < '4'
+mako==1.3.3; python_version >= '3.8' and python_version < '4'
 markupsafe==2.1.5; python_version >= '3.8' and python_version < '4'
 marshmallow==3.21.1; python_version >= '3.8' and python_version < '4'
 maya==0.6.1; python_version >= '3.8' and python_version < '4'
 mnemonic==0.20; python_version >= '3.8' and python_version < '4'
 msgpack-python==0.5.6; python_version >= '3.8' and python_version < '4'
 multidict==6.0.5; python_version >= '3.8' and python_version < '4'
-nucypher@ git+https://github.com/nucypher/nucypher.git@4090a342aff2ea8a600c3694fc149bc57549483f
+nucypher@ git+https://github.com/nucypher/nucypher.git@498b9887a1b30c804bc6ca5955ddea8bea49d73a
 nucypher-core==0.13.0
 packaging==23.2; python_version >= '3.8' and python_version < '4'
 parsimonious==0.9.0; python_version >= '3.8' and python_version < '4'
 pendulum==3.0.0; python_version >= '3.8' and python_version < '4'
 prometheus-client==0.20.0; python_version >= '3.8'
 prometheus-flask-exporter==0.23.0
-protobuf==5.26.0rc2; python_version >= '3.8' and python_version < '4'
+protobuf==5.26.1; python_version >= '3.8' and python_version < '4'
 pyasn1==0.6.0; python_version >= '3.8' and python_version < '4'
 pyasn1-modules==0.4.0; python_version >= '3.8' and python_version < '4'
 pychalk==2.0.1; python_version >= '3.8' and python_version < '4'
@@ -66,23 +66,24 @@ pycryptodome==3.20.0; python_version >= '3.8' and python_version < '4'
 pynacl==1.5.0; python_version >= '3.8' and python_version < '4'
 pyopenssl==24.1.0; python_version >= '3.8' and python_version < '4'
 python-dateutil==2.9.0.post0; python_version >= '3.8' and python_version < '4'
+python-statemachine==2.1.2; python_version < '3.13' and python_version >= '3.8'
 pytz==2024.1; python_version >= '3.8' and python_version < '4'
 pyunormalize==15.1.0; python_version >= '3.8' and python_version < '4'
-referencing==0.33.0; python_version >= '3.8' and python_version < '4'
+referencing==0.34.0; python_version >= '3.8' and python_version < '4'
 regex==2023.12.25; python_version >= '3.8' and python_version < '4'
 requests==2.31.0; python_version >= '3.8' and python_version < '4'
 rlp==3.0.0; python_version >= '3.8' and python_version < '4'
-rpds-py==0.17.1; python_version >= '3.8' and python_version < '4'
+rpds-py==0.18.0; python_version >= '3.8' and python_version < '4'
 service-identity==24.1.0; python_version >= '3.8' and python_version < '4'
-setuptools==69.0.3; python_version >= '3.8' and python_version < '4'
+setuptools==69.2.0; python_version >= '3.8' and python_version < '4'
 six==1.16.0; python_version >= '3.8' and python_version < '4'
 snaptime==0.2.4; python_version >= '3.8' and python_version < '4'
 tabulate==0.9.0; python_version >= '3.8' and python_version < '4'
 time-machine==2.14.1; python_version >= '3.8' and python_version < '4'
 toolz==0.12.1; python_version >= '3.8' and python_version < '4'
-twisted==23.10.0; python_version >= '3.8' and python_version < '4'
+twisted==24.3.0; python_version >= '3.8' and python_version < '4'
 txaio==23.1.1; python_version >= '3.8' and python_version < '4'
-typing-extensions==4.9.0; python_version >= '3.8' and python_version < '4'
+typing-extensions==4.11.0; python_version >= '3.8' and python_version < '4'
 tzdata==2024.1; python_version >= '3.8' and python_version < '4'
 tzlocal==5.2; python_version >= '3.8' and python_version < '4'
 urllib3==2.2.0; python_version >= '3.8' and python_version < '4'
@@ -92,4 +93,4 @@ websockets==12.0; python_version >= '3.8' and python_version < '4'
 werkzeug==3.0.2; python_version >= '3.8' and python_version < '4'
 yarl==1.9.4; python_version >= '3.8' and python_version < '4'
 zipp==3.18.1; python_version < '3.10' and python_version >= '3.8'
-zope-interface==6.1; python_version >= '3.8' and python_version < '4'
+zope-interface==6.2; python_version >= '3.8' and python_version < '4'