key proposer preferences by slot and dependent root

eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/gossip/GossipExecutionPayloadBidTestExecutor.java

@@ -23,6 +23,7 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -236,7 +237,8 @@ public void onBlockImported(
 
     // acceptedPreferences tracks proposer preferences that have been accepted by the validator,
     // so the bid validator can look them up to check bid compatibility.
-    final Map<UInt64, ProposerPreferences> acceptedPreferences = new ConcurrentHashMap<>();
+    final Map<UInt64, Map<Bytes32, ProposerPreferences>> acceptedPreferences =
+        new ConcurrentHashMap<>();
     final ProposerPreferencesManager proposerPreferencesManager =
         new ProposerPreferencesManager() {
           @Override
@@ -252,8 +254,17 @@ public SafeFuture<InternalValidationResult> addRemote(
           }
 
           @Override
-          public Optional<ProposerPreferences> getProposerPreferences(final UInt64 slot) {
-            return Optional.ofNullable(acceptedPreferences.get(slot));
+          public Optional<ProposerPreferences> getProposerPreferences(
+              final UInt64 slot, final Bytes32 dependentRoot) {
+            return Optional.ofNullable(acceptedPreferences.get(slot))
+                .map(preferencesByDependentRoot -> preferencesByDependentRoot.get(dependentRoot));
+          }
+
+          @Override
+          public Collection<ProposerPreferences> getProposerPreferencesForSlot(final UInt64 slot) {
+            return Optional.ofNullable(acceptedPreferences.get(slot))
+                .map(preferencesByDependentRoot -> List.copyOf(preferencesByDependentRoot.values()))
+                .orElse(List.of());
           }
 
           @Override
@@ -284,8 +295,10 @@ public void subscribeOperationAdded(
                 proposerPreferencesSchema::sszDeserialize);
         result = safeJoin(preferencesValidator.validate(signedPreferences));
         if (result.isAccept()) {
-          acceptedPreferences.put(
-              signedPreferences.getMessage().getProposalSlot(), signedPreferences.getMessage());
+          final ProposerPreferences preferences = signedPreferences.getMessage();
+          acceptedPreferences
+              .computeIfAbsent(preferences.getProposalSlot(), __ -> new ConcurrentHashMap<>())
+              .put(preferences.getDependentRoot(), preferences);
         }
       } else if (messageName.startsWith("execution_payload_envelope_")) {
         final SignedExecutionPayloadEnvelope signedEnvelope =

ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/execution/DefaultProposerPreferencesManager.java

@@ -15,11 +15,14 @@
 
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentNavigableMap;
 import java.util.concurrent.ConcurrentSkipListMap;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.tuweni.bytes.Bytes32;
 import tech.pegasys.teku.ethereum.events.SlotEventsChannel;
 import tech.pegasys.teku.infrastructure.async.SafeFuture;
 import tech.pegasys.teku.infrastructure.subscribers.Subscribers;
@@ -40,8 +43,8 @@ public class DefaultProposerPreferencesManager
 
   private final ProposerPreferencesGossipValidator proposerPreferencesGossipValidator;
   private final PendingPool<PendingProposerPreferences> pendingProposerPreferences;
-  private final ConcurrentNavigableMap<UInt64, ProposerPreferences> acceptedProposerPreferences =
-      new ConcurrentSkipListMap<>();
+  private final ConcurrentNavigableMap<UInt64, Map<Bytes32, ProposerPreferences>>
+      acceptedProposerPreferences = new ConcurrentSkipListMap<>();
   private final Subscribers<OperationAddedSubscriber<SignedProposerPreferences>> subscribers =
       Subscribers.create(true);
 
@@ -65,8 +68,17 @@ public SafeFuture<InternalValidationResult> addRemote(
   }
 
   @Override
-  public Optional<ProposerPreferences> getProposerPreferences(final UInt64 slot) {
-    return Optional.ofNullable(acceptedProposerPreferences.get(slot));
+  public Optional<ProposerPreferences> getProposerPreferences(
+      final UInt64 slot, final Bytes32 dependentRoot) {
+    return Optional.ofNullable(acceptedProposerPreferences.get(slot))
+        .map(preferencesByDependentRoot -> preferencesByDependentRoot.get(dependentRoot));
+  }
+
+  @Override
+  public Collection<ProposerPreferences> getProposerPreferencesForSlot(final UInt64 slot) {
+    return Optional.ofNullable(acceptedProposerPreferences.get(slot))
+        .map(preferencesByDependentRoot -> List.copyOf(preferencesByDependentRoot.values()))
+        .orElse(List.of());
   }
 
   @Override
@@ -117,9 +129,10 @@ private void processValidationResult(
     switch (result.code()) {
       case ACCEPT -> {
         removePendingPreferences(signedProposerPreferences);
-        acceptedProposerPreferences.put(
-            signedProposerPreferences.getMessage().getProposalSlot(),
-            signedProposerPreferences.getMessage());
+        final ProposerPreferences proposerPreferences = signedProposerPreferences.getMessage();
+        acceptedProposerPreferences
+            .computeIfAbsent(proposerPreferences.getProposalSlot(), __ -> new ConcurrentHashMap<>())
+            .put(proposerPreferences.getDependentRoot(), proposerPreferences);
         subscribers.forEach(
             subscriber ->
                 subscriber.onOperationAdded(signedProposerPreferences, result, fromNetwork));

ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/execution/ProposerPreferencesManager.java

@@ -13,7 +13,10 @@
 
 package tech.pegasys.teku.statetransition.execution;
 
+import java.util.Collection;
+import java.util.List;
 import java.util.Optional;
+import org.apache.tuweni.bytes.Bytes32;
 import tech.pegasys.teku.infrastructure.async.SafeFuture;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.spec.datastructures.epbs.versions.gloas.ProposerPreferences;
@@ -38,10 +41,16 @@ public SafeFuture<InternalValidationResult> addRemote(
         }
 
         @Override
-        public Optional<ProposerPreferences> getProposerPreferences(final UInt64 slot) {
+        public Optional<ProposerPreferences> getProposerPreferences(
+            final UInt64 slot, final Bytes32 dependentRoot) {
           return Optional.empty();
         }
 
+        @Override
+        public Collection<ProposerPreferences> getProposerPreferencesForSlot(final UInt64 slot) {
+          return List.of();
+        }
+
         @Override
         public void subscribeOperationAdded(
             final OperationAddedSubscriber<SignedProposerPreferences> subscriber) {}
@@ -53,7 +62,9 @@ SafeFuture<InternalValidationResult> addLocal(
   SafeFuture<InternalValidationResult> addRemote(
       SignedProposerPreferences signedProposerPreferences);
 
-  Optional<ProposerPreferences> getProposerPreferences(UInt64 slot);
+  Optional<ProposerPreferences> getProposerPreferences(UInt64 slot, Bytes32 dependentRoot);
+
+  Collection<ProposerPreferences> getProposerPreferencesForSlot(UInt64 slot);
 
   void subscribeOperationAdded(OperationAddedSubscriber<SignedProposerPreferences> subscriber);
 }

ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/forkchoice/ProposersDataManager.java

@@ -376,10 +376,10 @@ UInt64 getTargetGasLimit(
       final UInt64 proposerIndex,
       final Optional<SignedValidatorRegistration> validatorRegistration) {
     // post-Gloas, we use signed proposer preferences
-    return proposerPreferencesManager
-        .getProposerPreferences(blockSlot)
+    return proposerPreferencesManager.getProposerPreferencesForSlot(blockSlot).stream()
         .filter(
             proposerPreferences -> proposerPreferences.getValidatorIndex().equals(proposerIndex))
+        .findFirst()
         .map(ProposerPreferences::getTargetGasLimit)
         // pre-Gloas, we use validator registrations
         .or(

ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/validation/ExecutionPayloadBidGossipValidator.java

@@ -113,113 +113,6 @@ public SafeFuture<InternalValidationResult> validate(
       }
     }
 
-    /*
-     * [IGNORE] the SignedProposerPreferences where preferences.proposal_slot is equal to
-     * bid.slot has been seen
-     */
-    final Optional<ProposerPreferences> proposerPreferences =
-        proposerPreferencesManager.getProposerPreferences(bid.getSlot());
-    if (proposerPreferences.isEmpty()) {
-      return completedFuture(
-          saveBidForFuture(bid, "no proposer preferences available; saving for future processing"));
-    }
-
-    /*
-     * [REJECT] bid.fee_recipient matches the fee_recipient from the proposer's
-     * SignedProposerPreferences associated with bid.slot
-     */
-    if (!bid.getFeeRecipient().equals(proposerPreferences.get().getFeeRecipient())) {
-      return completedFuture(
-          ignoreBid(
-              bid,
-              "fee recipient %s does not match proposer preferences fee recipient %s",
-              bid.getFeeRecipient(),
-              proposerPreferences.get().getFeeRecipient()));
-    }
-
-    /*
-     * [IGNORE] this is the first signed bid seen with a valid signature from the given builder for the tuple
-     * (bid.slot, bid.parent_block_hash, bid.parent_block_root).
-     */
-    final BuilderAndParent builderAndParent =
-        new BuilderAndParent(
-            bid.getBuilderIndex(), bid.getParentBlockHash(), bid.getParentBlockRoot());
-    if (seenExecutionPayloadBids.getOrDefault(bid.getSlot(), Set.of()).contains(builderAndParent)) {
-      return completedFuture(
-          ignoreBid(
-              bid,
-              "already received for parent block hash %s and parent block root %s",
-              bid.getParentBlockHash(),
-              bid.getParentBlockRoot()));
-    }
-
-    /*
-     * [IGNORE] this bid is the highest value bid seen for the tuple
-     * (bid.slot, bid.parent_block_hash, bid.parent_block_root).
-     *
-     * Note: Implementations SHOULD include DoS prevention measures to
-     * mitigate spam from malicious builders submitting numerous bids with minimal value increments.
-     * Possible strategies include: (1) only forwarding bids that exceed the current highest bid by a
-     * minimum threshold, or (2) forwarding only the highest observed bid at regular time intervals.
-     *
-     */
-    final BidParent bidValueKey =
-        new BidParent(bid.getSlot(), bid.getParentBlockHash(), bid.getParentBlockRoot());
-    final UInt64 existingBidValue = highestBids.getOrDefault(bidValueKey, UInt64.ZERO);
-    if (!existingBidValue.isZero()) {
-      final UInt64 minRequiredBid = calculateMinimumRequiredBid(existingBidValue);
-
-      if (bid.getValue().isLessThan(minRequiredBid)) {
-        return completedFuture(
-            ignoreBid(
-                bid,
-                "does not meet minimum increment threshold (%s%%); current highest is %s ETH and minimum required is %s ETH",
-                minBidIncrementPercentage,
-                gweiToEth(existingBidValue),
-                gweiToEth(minRequiredBid)));
-      }
-    }
-
-    /*
-     * [IGNORE] bid.parent_block_hash is the block hash of a known execution payload in fork choice
-     * and is_gas_limit_target_compatible(parent_gas_limit, bid.gas_limit, proposer_preferences.target_gas_limit)
-     * is True where parent_gas_limit is the gas_limit of that execution payload.
-     */
-    final Optional<UInt64> maybeParentGasLimit =
-        gossipValidationHelper.getGasLimitForExecutionPayload(
-            bid.getParentBlockRoot(), bid.getParentBlockHash());
-    if (maybeParentGasLimit.isEmpty()) {
-      return completedFuture(
-          saveBidForFuture(
-              bid,
-              "parent execution payload gas limit is unavailable for parent block hash %s; saving for future processing",
-              bid.getParentBlockHash()));
-    }
-    final UInt64 parentGasLimit = maybeParentGasLimit.get();
-    final UInt64 targetGasLimit = proposerPreferences.get().getTargetGasLimit();
-    if (!isGasLimitTargetCompatible(parentGasLimit, bid.getGasLimit(), targetGasLimit)) {
-      return completedFuture(
-          ignoreBid(
-              bid,
-              "gas limit %s is not compatible with parent gas limit %s and proposer preferences target gas limit %s",
-              bid.getGasLimit(),
-              parentGasLimit,
-              targetGasLimit));
-    }
-
-    /*
-     * [IGNORE] The bid is compatible with the current head branch, i.e.
-     * is_bid_compatible_with_head(store, bid) returns True.
-     */
-    if (!gossipValidationHelper.isBidCompatibleWithHead(bid)) {
-      return completedFuture(
-          ignoreBid(
-              bid,
-              "is not compatible with the current head branch (parent block hash %s, parent block root %s)",
-              bid.getParentBlockHash(),
-              bid.getParentBlockRoot()));
-    }
-
     /*
      * Retrieve the bid's parent block slot for the remaining validation rules.
      */
@@ -259,6 +152,116 @@ public SafeFuture<InternalValidationResult> validate(
               }
               final BeaconState state = maybeState.get();
 
+              /*
+               * [IGNORE] The matching proposer preferences have been seen
+               */
+              final Optional<Bytes32> maybeDependentRoot =
+                  gossipValidationHelper.getShufflingDependentRoot(
+                      bid.getParentBlockRoot(), bid.getSlot());
+              if (maybeDependentRoot.isEmpty()) {
+                return saveBidForFuture(
+                    bid, "shuffling dependent root is unavailable; saving for future processing");
+              }
+              final Optional<ProposerPreferences> proposerPreferences =
+                  proposerPreferencesManager.getProposerPreferences(
+                      bid.getSlot(), maybeDependentRoot.get());
+              if (proposerPreferences.isEmpty()) {
+                return saveBidForFuture(
+                    bid, "no proposer preferences available; saving for future processing");
+              }
+
+              /*
+               * [REJECT] bid.fee_recipient matches the fee_recipient from the proposer's
+               * SignedProposerPreferences associated with bid.slot
+               */
+              if (!bid.getFeeRecipient().equals(proposerPreferences.get().getFeeRecipient())) {
+                return ignoreBid(
+                    bid,
+                    "fee recipient %s does not match proposer preferences fee recipient %s",
+                    bid.getFeeRecipient(),
+                    proposerPreferences.get().getFeeRecipient());
+              }
+
+              /*
+               * [IGNORE] this is the first signed bid seen with a valid signature from the given builder for the tuple
+               * (bid.slot, bid.parent_block_hash, bid.parent_block_root).
+               */
+              final BuilderAndParent builderAndParent =
+                  new BuilderAndParent(
+                      bid.getBuilderIndex(), bid.getParentBlockHash(), bid.getParentBlockRoot());
+              if (seenExecutionPayloadBids
+                  .getOrDefault(bid.getSlot(), Set.of())
+                  .contains(builderAndParent)) {
+                return ignoreBid(
+                    bid,
+                    "already received for parent block hash %s and parent block root %s",
+                    bid.getParentBlockHash(),
+                    bid.getParentBlockRoot());
+              }
+
+              /*
+               * [IGNORE] this bid is the highest value bid seen for the tuple
+               * (bid.slot, bid.parent_block_hash, bid.parent_block_root).
+               *
+               * Note: Implementations SHOULD include DoS prevention measures to
+               * mitigate spam from malicious builders submitting numerous bids with minimal value increments.
+               * Possible strategies include: (1) only forwarding bids that exceed the current highest bid by a
+               * minimum threshold, or (2) forwarding only the highest observed bid at regular time intervals.
+               *
+               */
+              final BidParent bidValueKey =
+                  new BidParent(bid.getSlot(), bid.getParentBlockHash(), bid.getParentBlockRoot());
+              final UInt64 existingBidValue = highestBids.getOrDefault(bidValueKey, UInt64.ZERO);
+              if (!existingBidValue.isZero()) {
+                final UInt64 minRequiredBid = calculateMinimumRequiredBid(existingBidValue);
+
+                if (bid.getValue().isLessThan(minRequiredBid)) {
+                  return ignoreBid(
+                      bid,
+                      "does not meet minimum increment threshold (%s%%); current highest is %s ETH and minimum required is %s ETH",
+                      minBidIncrementPercentage,
+                      gweiToEth(existingBidValue),
+                      gweiToEth(minRequiredBid));
+                }
+              }
+
+              /*
+               * [IGNORE] bid.parent_block_hash is the block hash of a known execution payload in fork choice
+               * and is_gas_limit_target_compatible(parent_gas_limit, bid.gas_limit, proposer_preferences.target_gas_limit)
+               * is True where parent_gas_limit is the gas_limit of that execution payload.
+               */
+              final Optional<UInt64> maybeParentGasLimit =
+                  gossipValidationHelper.getGasLimitForExecutionPayload(
+                      bid.getParentBlockRoot(), bid.getParentBlockHash());
+              if (maybeParentGasLimit.isEmpty()) {
+                return saveBidForFuture(
+                    bid,
+                    "parent execution payload gas limit is unavailable for parent block hash %s; saving for future processing",
+                    bid.getParentBlockHash());
+              }
+              final UInt64 parentGasLimit = maybeParentGasLimit.get();
+              final UInt64 targetGasLimit = proposerPreferences.get().getTargetGasLimit();
+              if (!isGasLimitTargetCompatible(parentGasLimit, bid.getGasLimit(), targetGasLimit)) {
+                return ignoreBid(
+                    bid,
+                    "gas limit %s is not compatible with parent gas limit %s and proposer preferences target gas limit %s",
+                    bid.getGasLimit(),
+                    parentGasLimit,
+                    targetGasLimit);
+              }
+
+              /*
+               * [IGNORE] The bid is compatible with the current head branch, i.e.
+               * is_bid_compatible_with_head(store, bid) returns True.
+               */
+              if (!gossipValidationHelper.isBidCompatibleWithHead(bid)) {
+                return ignoreBid(
+                    bid,
+                    "is not compatible with the current head branch (parent block hash %s, parent block root %s)",
+                    bid.getParentBlockHash(),
+                    bid.getParentBlockRoot());
+              }
+
               /*
                * [REJECT] bid.prev_randao is the correct RANDAO mix -- i.e. validate that
                * bid.prev_randao == get_randao_mix(parent_state, get_current_epoch(parent_state)).