op-acceptance-tests: Cache and canonicalize L2EL payloads after gap fill (#17675)

* op-acceptance-tests: Cache and canonicalize L2EL payloads after gap fill

* Clarify args for retrying

* better var name

Author: pcw109550

op-acceptance-tests/tests/elsync/elsync_test.go

@@ -0,0 +1,52 @@
+package elsync
+
+import (
+	"testing"
+
+	"github.com/ethereum-optimism/optimism/op-devstack/devtest"
+	"github.com/ethereum-optimism/optimism/op-devstack/presets"
+	"github.com/ethereum-optimism/optimism/op-service/eth"
+	"github.com/ethereum-optimism/optimism/op-supervisor/supervisor/types"
+)
+
+func TestELSyncAfterInitialELSync(gt *testing.T) {
+	t := devtest.SerialT(gt)
+	sys := presets.NewSingleChainMultiNodeWithoutCheck(t)
+	require := t.Require()
+
+	sys.L2CL.Advanced(types.LocalUnsafe, 7, 30)
+
+	// batcher down so safe not advanced
+	require.Equal(uint64(0), sys.L2CL.HeadBlockRef(types.LocalSafe).Number)
+	require.Equal(uint64(0), sys.L2CLB.HeadBlockRef(types.LocalSafe).Number)
+	// verifier not advanced unsafe head
+	require.Equal(uint64(0), sys.L2CLB.HeadBlockRef(types.LocalUnsafe).Number)
+
+	// Finish EL sync by supplying the first block
+	// EL Sync finished because underlying EL has states to validate the payload for block 1
+	sys.L2CLB.SignalTarget(sys.L2EL, 1)
+
+	// Note: Below non-canonical payload caching behavior was only observed after the initial EL Sync has finished
+
+	// Send payloads for block 3, 4, 5, 7 which will make non-canonical blocks, block 2 missed
+	// Non-canonical payloads will be buffered at the L2EL
+	// Order does not matter
+	for _, target := range []uint64{5, 3, 4, 7} {
+		sys.L2CLB.SignalTarget(sys.L2EL, target)
+		// Canonical unsafe head never advances because of the gap
+		require.Equal(uint64(1), sys.L2ELB.BlockRefByLabel(eth.Unsafe).Number)
+	}
+
+	// Send missing gap, payload 2
+	sys.L2CLB.SignalTarget(sys.L2EL, 2)
+
+	retries := 2
+	// Gap filled and payload 2, 3, 4, 5 became canonical. Payload 7 is still non canonical
+	sys.L2ELB.Reached(eth.Unsafe, 5, retries)
+
+	// Send missing gap, payload 6
+	sys.L2CLB.SignalTarget(sys.L2EL, 6)
+
+	// Gap filled and block 6, 7 became canonical
+	sys.L2ELB.Reached(eth.Unsafe, 7, retries)
+}

op-acceptance-tests/tests/elsync/init_test.go

@@ -0,0 +1,23 @@
+package elsync
+
+import (
+	"testing"
+
+	bss "github.com/ethereum-optimism/optimism/op-batcher/batcher"
+	"github.com/ethereum-optimism/optimism/op-devstack/compat"
+	"github.com/ethereum-optimism/optimism/op-devstack/presets"
+	"github.com/ethereum-optimism/optimism/op-devstack/stack"
+	"github.com/ethereum-optimism/optimism/op-devstack/sysgo"
+)
+
+func TestMain(m *testing.M) {
+	// No ELP2P, CLP2P to control the supply of unsafe payload to the CL
+	presets.DoMain(m, presets.WithSingleChainMultiNodeWithoutP2P(),
+		presets.WithExecutionLayerSyncOnVerifiers(),
+		presets.WithCompatibleTypes(compat.SysGo),
+		stack.MakeCommon(sysgo.WithBatcherOption(func(id stack.L2BatcherID, cfg *bss.CLIConfig) {
+			// For stopping derivation, not to advance safe heads
+			cfg.Stopped = true
+		})),
+	)
+}

op-devstack/dsl/l2_el.go

@@ -90,6 +90,23 @@ func (el *L2ELNode) NotAdvancedFn(label eth.BlockLabel) CheckFunc {
 	}
 }
 
+func (el *L2ELNode) ReachedFn(label eth.BlockLabel, target uint64, attempts int) CheckFunc {
+	return func() error {
+		logger := el.log.With("id", el.inner.ID(), "chain", el.ChainID(), "label", label, "target", target)
+		logger.Info("Expecting L2EL to reach")
+		return retry.Do0(el.ctx, attempts, &retry.FixedStrategy{Dur: 2 * time.Second},
+			func() error {
+				head := el.BlockRefByLabel(label)
+				if head.Number >= target {
+					logger.Info("L2EL advanced", "target", target)
+					return nil
+				}
+				logger.Info("L2EL sync status", "current", head.Number)
+				return fmt.Errorf("expected head to advance: %s", label)
+			})
+	}
+}
+
 func (el *L2ELNode) BlockRefByNumber(num uint64) eth.L2BlockRef {
 	ctx, cancel := context.WithTimeout(el.ctx, DefaultTimeout)
 	defer cancel()
@@ -135,6 +152,10 @@ func (el *L2ELNode) Advanced(label eth.BlockLabel, block uint64) {
 	el.require.NoError(el.AdvancedFn(label, block)())
 }
 
+func (el *L2ELNode) Reached(label eth.BlockLabel, block uint64, attempts int) {
+	el.require.NoError(el.ReachedFn(label, block, attempts)())
+}
+
 func (el *L2ELNode) NotAdvanced(label eth.BlockLabel) {
 	el.require.NoError(el.NotAdvancedFn(label)())
 }

op-devstack/presets/singlechain_multinode.go

@@ -22,6 +22,16 @@ func WithSingleChainMultiNode() stack.CommonOption {
 }
 
 func NewSingleChainMultiNode(t devtest.T) *SingleChainMultiNode {
+	preset := NewSingleChainMultiNodeWithoutCheck(t)
+	// Ensure the follower node is in sync with the sequencer before starting tests
+	dsl.CheckAll(t,
+		preset.L2CLB.MatchedFn(preset.L2CL, types.CrossSafe, 30),
+		preset.L2CLB.MatchedFn(preset.L2CL, types.LocalUnsafe, 30),
+	)
+	return preset
+}
+
+func NewSingleChainMultiNodeWithoutCheck(t devtest.T) *SingleChainMultiNode {
 	system := shim.NewSystem(t)
 	orch := Orchestrator()
 	orch.Hydrate(system)
@@ -41,10 +51,9 @@ func NewSingleChainMultiNode(t devtest.T) *SingleChainMultiNode {
 		L2ELB:   dsl.NewL2ELNode(verifierEL, orch.ControlPlane()),
 		L2CLB:   dsl.NewL2CLNode(verifierCL, orch.ControlPlane()),
 	}
-	// Ensure the follower node is in sync with the sequencer before starting tests
-	dsl.CheckAll(t,
-		preset.L2CLB.MatchedFn(preset.L2CL, types.CrossSafe, 30),
-		preset.L2CLB.MatchedFn(preset.L2CL, types.LocalUnsafe, 30),
-	)
 	return preset
 }
+
+func WithSingleChainMultiNodeWithoutP2P() stack.CommonOption {
+	return stack.MakeCommon(sysgo.DefaultSingleChainMultiNodeSystemWithoutP2P(&sysgo.DefaultSingleChainMultiNodeSystemIDs{}))
+}

op-devstack/sysgo/system_singlechain_multinode.go

@@ -39,3 +39,18 @@ func DefaultSingleChainMultiNodeSystem(dest *DefaultSingleChainMultiNodeSystemID
 	}))
 	return opt
 }
+
+func DefaultSingleChainMultiNodeSystemWithoutP2P(dest *DefaultSingleChainMultiNodeSystemIDs) stack.Option[*Orchestrator] {
+	ids := NewDefaultSingleChainMultiNodeSystemIDs(DefaultL1ID, DefaultL2AID)
+
+	opt := stack.Combine[*Orchestrator]()
+	opt.Add(DefaultMinimalSystem(&dest.DefaultMinimalSystemIDs))
+
+	opt.Add(WithL2ELNode(ids.L2ELB))
+	opt.Add(WithL2CLNode(ids.L2CLB, ids.L1CL, ids.L1EL, ids.L2ELB))
+
+	opt.Add(stack.Finally(func(orch *Orchestrator) {
+		*dest = ids
+	}))
+	return opt
+}