Merge pull request #5887 from Algo-devops-service/relstable3.21.0

.aptly.conf

@@ -1,5 +1,4 @@
 {
-  "rootDir": "/root/aptly",
   "downloadConcurrency": 4,
   "downloadSpeedLimit": 0,
   "architectures": [],
@@ -27,4 +26,3 @@
   },
   "SwiftPublishEndpoints": {}
 }
-

agreement/actions.go

@@ -232,8 +232,7 @@ type ensureAction struct {
 	Payload proposal
 	// the certificate proving commitment
 	Certificate Certificate
-
-	// The time that the winning proposal-vote was validated, relative to the beginning of the round
+	// The time that the winning proposal-vote was validated for round credentialRoundLag back from the current one
 	voteValidatedAt time.Duration
 	// The dynamic filter timeout calculated for this round, even if not enabled, for reporting to telemetry.
 	dynamicFilterTimeout time.Duration

agreement/dynamicFilterTimeoutParams.go

@@ -31,7 +31,7 @@ const dynamicFilterCredentialArrivalHistory int = 40
 
 // DynamicFilterTimeoutLowerBound specifies a minimal duration that the
 // filter timeout must meet.
-const dynamicFilterTimeoutLowerBound time.Duration = 500 * time.Millisecond
+const dynamicFilterTimeoutLowerBound time.Duration = 2500 * time.Millisecond
 
 // DynamicFilterTimeoutCredentialArrivalHistoryIdx specified which sample to use
 // out of a sorted DynamicFilterCredentialArrivalHistory-sized array of time

agreement/player.go

@@ -100,10 +100,19 @@ func (p *player) handle(r routerHandle, e event) []action {
 			r.t.logTimeout(*p)
 		}
 
+		var deadlineTimeout time.Duration
+		if e.Proto.Version == "" || e.Proto.Err != nil {
+			r.t.log.Errorf("failed to read valid protocol version for timeout event (proto %v): %v. "+
+				"Falling Back to default deadline timeout.", e.Proto.Version, e.Proto.Err)
+			deadlineTimeout = DefaultDeadlineTimeout()
+		} else {
+			deadlineTimeout = DeadlineTimeout(p.Period, e.Proto.Version)
+		}
+
 		switch p.Step {
 		case soft:
 			// precondition: nap = false
-			actions = p.issueSoftVote(r)
+			actions = p.issueSoftVote(r, deadlineTimeout)
 			p.Step = cert
 			// update tracer state to match player
 			r.t.setMetadata(tracerMetadata{p.Round, p.Period, p.Step})
@@ -113,16 +122,16 @@ func (p *player) handle(r routerHandle, e event) []action {
 			p.Step = next
 			// update tracer state to match player
 			r.t.setMetadata(tracerMetadata{p.Round, p.Period, p.Step})
-			return p.issueNextVote(r)
+			return p.issueNextVote(r, deadlineTimeout)
 		default:
 			if p.Napping {
-				return p.issueNextVote(r) // sets p.Napping to false
+				return p.issueNextVote(r, deadlineTimeout) // sets p.Napping to false
 			}
 			// not napping, so we should enter a new step
 			p.Step++ // note: this must happen before next timeout setting.
 			// TODO add unit test to ensure that deadlines increase monotonically here
 
-			lower, upper := p.Step.nextVoteRanges()
+			lower, upper := p.Step.nextVoteRanges(deadlineTimeout)
 			delta := time.Duration(e.RandomEntropy % uint64(upper-lower))
 
 			p.Napping = true
@@ -158,7 +167,7 @@ func (p *player) handleFastTimeout(r routerHandle, e timeoutEvent) []action {
 	return p.issueFastVote(r)
 }
 
-func (p *player) issueSoftVote(r routerHandle) (actions []action) {
+func (p *player) issueSoftVote(r routerHandle, deadlineTimeout time.Duration) (actions []action) {
 	defer func() {
 		p.Deadline = Deadline{Duration: deadlineTimeout, Type: TimeoutDeadline}
 	}()
@@ -202,7 +211,7 @@ func (p *player) issueCertVote(r routerHandle, e committableEvent) action {
 	return pseudonodeAction{T: attest, Round: p.Round, Period: p.Period, Step: cert, Proposal: e.Proposal}
 }
 
-func (p *player) issueNextVote(r routerHandle) []action {
+func (p *player) issueNextVote(r routerHandle, deadlineTimeout time.Duration) []action {
 	actions := p.partitionPolicy(r)
 
 	a := pseudonodeAction{T: attest, Round: p.Round, Period: p.Period, Step: p.Step, Proposal: bottom}
@@ -226,7 +235,7 @@ func (p *player) issueNextVote(r routerHandle) []action {
 
 	r.t.timeR().RecStep(p.Period, p.Step, a.Proposal)
 
-	_, upper := p.Step.nextVoteRanges()
+	_, upper := p.Step.nextVoteRanges(deadlineTimeout)
 	p.Napping = false
 	p.Deadline = Deadline{Duration: upper, Type: TimeoutDeadline}
 	return actions

agreement/player_permutation_test.go

@@ -811,17 +811,16 @@ func TestPlayerPermutation(t *testing.T) {
 }
 
 func playerPermutationCheck(t *testing.T, enableDynamicFilterTimeout bool) {
-	// create a protocol version where dynamic filter is enabled
-	version, _, configCleanup := createDynamicFilterConfig()
+	// create a protocol where dynamic filter is set based on the enableDynamicFilterTimeout flag
+	dynamicFilterOverriddenProtocol, _, configCleanup := overrideConfigWithDynamicFilterParam(enableDynamicFilterTimeout)
 	defer configCleanup()
 
 	for i := 0; i < 7; i++ {
 		for j := 0; j < 14; j++ {
 			_, pMachine, helper := getPlayerPermutation(t, i)
 			inMsg := getMessageEventPermutation(t, j, helper)
-			if enableDynamicFilterTimeout {
-				inMsg.Proto = ConsensusVersionView{Version: version}
-			}
+			inMsg.Proto = ConsensusVersionView{Version: dynamicFilterOverriddenProtocol}
+
 			err, panicErr := pMachine.transition(inMsg)
 			fmt.Println(pMachine.getTrace().events)
 			fmt.Println("")

agreement/player_test.go

@@ -37,7 +37,7 @@ func init() {
 }
 
 func makeTimeoutEvent() timeoutEvent {
-	return timeoutEvent{T: timeout, RandomEntropy: crypto.RandUint64()}
+	return timeoutEvent{T: timeout, RandomEntropy: crypto.RandUint64(), Proto: ConsensusVersionView{Version: protocol.ConsensusCurrentVersion}}
 }
 
 func generateProposalEvents(t *testing.T, player player, accs testAccountData, f testBlockFactory, ledger Ledger) (voteBatch []event, payloadBatch []event, lowestProposal proposalValue) {
@@ -3240,7 +3240,7 @@ func TestPlayerAlwaysResynchsPinnedValue(t *testing.T) {
 func TestPlayerRetainsReceivedValidatedAtOneSample(t *testing.T) {
 	partitiontest.PartitionTest(t)
 
-	version := protocol.ConsensusFuture
+	version := protocol.ConsensusCurrentVersion
 	const r = round(20239)
 	const p = period(131)
 	pWhite, pM, helper := setupP(t, r-1, p, soft)
@@ -3262,7 +3262,7 @@ func TestPlayerRetainsReceivedValidatedAtOneSample(t *testing.T) {
 func TestPlayerRetainsReceivedValidatedAtCredentialHistory(t *testing.T) {
 	partitiontest.PartitionTest(t)
 
-	version := protocol.ConsensusFuture
+	version := protocol.ConsensusCurrentVersion
 	const r = round(20239)
 	const p = period(0)
 	pWhite, pM, helper := setupP(t, r-credentialRoundLag-1, p, soft)
@@ -3301,7 +3301,7 @@ func TestPlayerRetainsReceivedValidatedAtCredentialHistory(t *testing.T) {
 func TestPlayerRetainsEarlyReceivedValidatedAtOneSample(t *testing.T) {
 	partitiontest.PartitionTest(t)
 
-	version := protocol.ConsensusFuture
+	version := protocol.ConsensusCurrentVersion
 	const r = round(20239)
 	const p = period(0)
 	pWhite, pM, helper := setupP(t, r-1, p, soft)
@@ -3339,7 +3339,7 @@ func testClockForRound(t *testing.T, pWhite *player, fixedDur time.Duration, cur
 func TestPlayerRetainsLateReceivedValidatedAtOneSample(t *testing.T) {
 	partitiontest.PartitionTest(t)
 
-	version := protocol.ConsensusFuture
+	version := protocol.ConsensusCurrentVersion
 	const r = round(20239)
 	const p = period(0)
 	pWhite, pM, helper := setupP(t, r-1, p, soft)
@@ -3382,7 +3382,7 @@ func TestPlayerRetainsReceivedValidatedAtForHistoryWindowLateBetter(t *testing.T
 }
 
 func testPlayerRetainsReceivedValidatedAtForHistoryWindow(t *testing.T, addBetterLate bool) {
-	version := protocol.ConsensusFuture
+	version := protocol.ConsensusCurrentVersion
 	const r = round(20239)
 	const p = period(0)
 	pWhite, pM, helper := setupP(t, r-1, p, soft)
@@ -3449,7 +3449,7 @@ func testPlayerRetainsReceivedValidatedAtForHistoryWindow(t *testing.T, addBette
 func TestPlayerRetainsReceivedValidatedAtPPOneSample(t *testing.T) {
 	partitiontest.PartitionTest(t)
 
-	version, _, configCleanup := createDynamicFilterConfig()
+	version, _, configCleanup := overrideConfigWithDynamicFilterParam(true)
 	defer configCleanup()
 	const r = round(20239)
 	const p = period(0)
@@ -3505,7 +3505,7 @@ func TestPlayerRetainsReceivedValidatedAtPPOneSample(t *testing.T) {
 func TestPlayerRetainsEarlyReceivedValidatedAtPPOneSample(t *testing.T) {
 	partitiontest.PartitionTest(t)
 
-	version, _, configCleanup := createDynamicFilterConfig()
+	version, _, configCleanup := overrideConfigWithDynamicFilterParam(true)
 	defer configCleanup()
 
 	const r = round(20239)
@@ -3559,7 +3559,7 @@ func TestPlayerRetainsEarlyReceivedValidatedAtPPOneSample(t *testing.T) {
 func TestPlayerRetainsLateReceivedValidatedAtPPOneSample(t *testing.T) {
 	partitiontest.PartitionTest(t)
 
-	version, _, configCleanup := createDynamicFilterConfig()
+	version, _, configCleanup := overrideConfigWithDynamicFilterParam(true)
 	defer configCleanup()
 	const r = round(20239)
 	const p = period(0)
@@ -3613,7 +3613,7 @@ func TestPlayerRetainsLateReceivedValidatedAtPPOneSample(t *testing.T) {
 func TestPlayerRetainsReceivedValidatedAtPPForHistoryWindow(t *testing.T) {
 	partitiontest.PartitionTest(t)
 
-	version := protocol.ConsensusFuture
+	version := protocol.ConsensusCurrentVersion
 	const r = round(20239)
 	const p = period(0)
 	pWhite, pM, helper := setupP(t, r-1, p, soft)
@@ -3655,7 +3655,7 @@ func TestPlayerRetainsReceivedValidatedAtAVPPOneSample(t *testing.T) {
 	partitiontest.PartitionTest(t)
 
 	// create a protocol version where dynamic lambda is enabled
-	version, _, configCleanup := createDynamicFilterConfig()
+	version, _, configCleanup := overrideConfigWithDynamicFilterParam(true)
 	defer configCleanup()
 	const r = round(20239)
 	const p = period(0)
@@ -3710,7 +3710,7 @@ func TestPlayerRetainsReceivedValidatedAtAVPPOneSample(t *testing.T) {
 func TestPlayerRetainsEarlyReceivedValidatedAtAVPPOneSample(t *testing.T) {
 	partitiontest.PartitionTest(t)
 
-	version := protocol.ConsensusFuture
+	version := protocol.ConsensusCurrentVersion
 	const r = round(20239)
 	const p = period(0)
 	pWhite, pM, helper := setupP(t, r-1, p, soft)
@@ -3729,7 +3729,7 @@ func TestPlayerRetainsEarlyReceivedValidatedAtAVPPOneSample(t *testing.T) {
 	require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0)
 
 	// create a protocol version where dynamic filter is enabled
-	version, _, configCleanup := createDynamicFilterConfig()
+	version, _, configCleanup := overrideConfigWithDynamicFilterParam(true)
 	defer configCleanup()
 
 	// send votePresent message (mimicking the first AV message validating)
@@ -3767,7 +3767,7 @@ func TestPlayerRetainsEarlyReceivedValidatedAtAVPPOneSample(t *testing.T) {
 func TestPlayerRetainsLateReceivedValidatedAtAVPPOneSample(t *testing.T) {
 	partitiontest.PartitionTest(t)
 
-	version := protocol.ConsensusFuture
+	version := protocol.ConsensusCurrentVersion
 	const r = round(20239)
 	const p = period(0)
 	pWhite, pM, helper := setupP(t, r-1, p, soft)
@@ -3786,7 +3786,7 @@ func TestPlayerRetainsLateReceivedValidatedAtAVPPOneSample(t *testing.T) {
 	require.Equal(t, pWhite.lowestCredentialArrivals.writePtr, 0)
 
 	// create a protocol version where dynamic filter is enabled
-	version, _, configCleanup := createDynamicFilterConfig()
+	version, _, configCleanup := overrideConfigWithDynamicFilterParam(true)
 	defer configCleanup()
 
 	// send votePresent message (mimicking the first AV message validating)
@@ -3821,7 +3821,7 @@ func TestPlayerRetainsLateReceivedValidatedAtAVPPOneSample(t *testing.T) {
 func TestPlayerRetainsReceivedValidatedAtAVPPHistoryWindow(t *testing.T) {
 	partitiontest.PartitionTest(t)
 
-	version := protocol.ConsensusFuture
+	version := protocol.ConsensusCurrentVersion
 	const r = round(20239)
 	const p = period(0)
 	pWhite, pM, helper := setupP(t, r-1, p, soft)

agreement/router.go

@@ -59,7 +59,16 @@ var credentialRoundLag round
 
 func init() {
 	// credential arrival time should be at most 2*config.Protocol.SmallLambda after it was sent
+	// Note that the credentialRoundLag is inversely proportional to the dynamicFilterTimeoutLowerBound
+	// in the default formula. Since we are adjusting this lower bound over time,
+	// for consistency in analytics we are setting the minimum to be 8 rounds
+	// (equivalent to a dynamicFilterTimeoutLowerBound of 500 ms).
+	minCredentialRoundLag := round(8) // round 2*2000ms / 500ms
 	credentialRoundLag = round(2 * config.Protocol.SmallLambda / dynamicFilterTimeoutLowerBound)
+
+	if credentialRoundLag < minCredentialRoundLag {
+		credentialRoundLag = minCredentialRoundLag
+	}
 	if credentialRoundLag*round(dynamicFilterTimeoutLowerBound) < round(2*config.Protocol.SmallLambda) {
 		credentialRoundLag++
 	}