fix: Add support for delayedack middleware to also delay ack and time…

…out packets (dymensionxyz#480)

go.mod

@@ -180,8 +180,8 @@ require (
 	github.com/tklauser/numcpus v0.6.0 // indirect
 	github.com/tyler-smith/go-bip39 v1.1.0 // indirect
 	github.com/ulikunitz/xz v0.5.10 // indirect
-	github.com/zondax/hid v0.9.1 // indirect
-	github.com/zondax/ledger-go v0.14.1 // indirect
+	github.com/zondax/hid v0.9.2 // indirect
+	github.com/zondax/ledger-go v0.14.3 // indirect
 	go.etcd.io/bbolt v1.3.6 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/otel v1.19.0 // indirect

go.sum

@@ -390,8 +390,8 @@ github.com/cosmos/iavl v0.19.6 h1:XY78yEeNPrEYyNCKlqr9chrwoeSDJ0bV2VjocTk//OU=
 github.com/cosmos/iavl v0.19.6/go.mod h1:X9PKD3J0iFxdmgNLa7b2LYWdsGd90ToV5cAONApkEPw=
 github.com/cosmos/ibc-go/v6 v6.2.1 h1:NiaDXTRhKwf3n9kELD4VRIe5zby1yk1jBvaz9tXTQ6k=
 github.com/cosmos/ibc-go/v6 v6.2.1/go.mod h1:XLsARy4Y7+GtAqzMcxNdlQf6lx+ti1e8KcMGv5NIK7A=
-github.com/cosmos/ledger-cosmos-go v0.12.2 h1:/XYaBlE2BJxtvpkHiBm97gFGSGmYGKunKyF3nNqAXZA=
-github.com/cosmos/ledger-cosmos-go v0.12.2/go.mod h1:ZcqYgnfNJ6lAXe4HPtWgarNEY+B74i+2/8MhZw4ziiI=
+github.com/cosmos/ledger-cosmos-go v0.12.4 h1:drvWt+GJP7Aiw550yeb3ON/zsrgW0jgh5saFCr7pDnw=
+github.com/cosmos/ledger-cosmos-go v0.12.4/go.mod h1:fjfVWRf++Xkygt9wzCsjEBdjcf7wiiY35fv3ctT+k4M=
 github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
@@ -1216,10 +1216,10 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-github.com/zondax/hid v0.9.1 h1:gQe66rtmyZ8VeGFcOpbuH3r7erYtNEAezCAYu8LdkJo=
-github.com/zondax/hid v0.9.1/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM=
-github.com/zondax/ledger-go v0.14.1 h1:Pip65OOl4iJ84WTpA4BKChvOufMhhbxED3BaihoZN4c=
-github.com/zondax/ledger-go v0.14.1/go.mod h1:fZ3Dqg6qcdXWSOJFKMG8GCTnD7slO/RL2feOQv8K320=
+github.com/zondax/hid v0.9.2 h1:WCJFnEDMiqGF64nlZz28E9qLVZ0KSJ7xpc5DLEyma2U=
+github.com/zondax/hid v0.9.2/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM=
+github.com/zondax/ledger-go v0.14.3 h1:wEpJt2CEcBJ428md/5MgSLsXLBos98sBOyxNmCjfUCw=
+github.com/zondax/ledger-go v0.14.3/go.mod h1:IKKaoxupuB43g4NxeQmbLXv7T9AlQyie1UpHb342ycI=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
 go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=

ibctesting/utils_test.go

@@ -49,7 +49,7 @@ func TestKeeperTestSuite(t *testing.T) {
 
 // SetupTest creates a coordinator with 2 test chains.
 func (suite *KeeperTestSuite) SetupTest() {
-	suite.coordinator = ibctesting.NewCoordinator(suite.T(), 3)               // initializes 2 test chains
+	suite.coordinator = ibctesting.NewCoordinator(suite.T(), 3)               // initializes 3 test chains
 	suite.hubChain = suite.coordinator.GetChain(ibctesting.GetChainID(1))     // convenience and readability
 	suite.cosmosChain = suite.coordinator.GetChain(ibctesting.GetChainID(2))  // convenience and readability
 	suite.rollappChain = suite.coordinator.GetChain(ibctesting.GetChainID(3)) // convenience and readability
@@ -127,7 +127,7 @@ func (suite *KeeperTestSuite) FinalizeRollapp() error {
 
 	// update the status of the stateInfo
 	rollappKeeper.SetStateInfo(ctx, stateInfo)
-	// uppdate the LatestStateInfoIndex of the rollapp
+	// update the LatestStateInfoIndex of the rollapp
 	rollappKeeper.SetLatestFinalizedStateIndex(ctx, stateInfoIdx)
 
 	err := rollappKeeper.GetHooks().AfterStateFinalized(

proto/dymension/delayedack/genesis.proto

@@ -2,10 +2,13 @@ syntax = "proto3";
 package dymensionxyz.dymension.delayedack;
 
 import "gogoproto/gogo.proto";
+import "dymension/delayedack/rollapp_packet.proto";
 
 
 option go_package = "github.com/dymensionxyz/dymension/v3/x/delayedack/types";
 
 // GenesisState defines the delayedack module's genesis state.
 message GenesisState {
+  // streams are all streams that should exist at genesis
+  repeated RollappPacket rollapp_packets = 2 [ (gogoproto.nullable) = false ];
 }

proto/dymension/delayedack/rollapp_packet.proto

@@ -7,15 +7,23 @@ import "ibc/core/channel/v1/channel.proto";
 option go_package = "github.com/dymensionxyz/dymension/v3/x/delayedack/types";
 
 message RollappPacket {
-    ibc.core.channel.v1.Packet packet = 1;
+    string rollapp_id = 1;
+    ibc.core.channel.v1.Packet packet = 2;
+    bytes acknowledgement = 3;
     enum Status {
       PENDING = 0;
       ACCEPTED = 1;
       REJECTED = 2;
     }
-    Status status = 2;
-    uint64 ProofHeight = 3;
-    string error = 4;
-    bytes relayer = 5;
+    Status status = 4;
+    uint64 ProofHeight = 5;
+    string error = 6;
+    bytes relayer = 7;
+    enum Type {
+      ON_RECV = 0;
+      ON_ACK = 1;
+      ON_TIMEOUT = 2;
+    }
+    Type type = 8;
 
   }

x/delayedack/genesis.go

@@ -8,11 +8,14 @@ import (
 
 // InitGenesis initializes the module's state from a provided genesis state.
 func InitGenesis(ctx sdk.Context, k keeper.Keeper, genState types.GenesisState) {
-	//FIXME: should we set the rollapp packets in store?
+	for _, packet := range genState.RollappPackets {
+		k.SetRollappPacket(ctx, packet)
+	}
 }
 
 // ExportGenesis returns the module's exported genesis
 func ExportGenesis(ctx sdk.Context, k keeper.Keeper) *types.GenesisState {
-	genesis := types.DefaultGenesis()
-	return genesis
+	return &types.GenesisState{
+		RollappPackets: k.GetAllRollappPackets(ctx),
+	}
 }

x/delayedack/hooks.go

@@ -2,8 +2,10 @@ package delayedack
 
 import (
 	sdk "github.com/cosmos/cosmos-sdk/types"
+	sdkerrors "github.com/cosmos/cosmos-sdk/types/errors"
 	"github.com/dymensionxyz/dymension/v3/x/delayedack/types"
 	rollapptypes "github.com/dymensionxyz/dymension/v3/x/rollapp/types"
+	"github.com/osmosis-labs/osmosis/v15/osmoutils"
 )
 
 var _ rollapptypes.RollappHooks = &IBCMiddleware{}
@@ -32,24 +34,74 @@ func (im IBCMiddleware) FinalizeRollappPackets(ctx sdk.Context, rollappID string
 	// Get the packets for the rollapp until height
 	logger.Debug("Finalizing IBC rollapp packets", "rollappID", rollappID, "state end height", stateEndHeight, "num packets", len(rollappPendingPackets))
 	for _, rollappPacket := range rollappPendingPackets {
-		logger.Debug("Finalizing IBC rollapp packet", "rollappID", rollappID, "sequence", rollappPacket.Packet.GetSequence(), "destination channel", rollappPacket.Packet.GetDestChannel())
+		logger.Debug("Finalizing IBC rollapp packet", "rollappID", rollappID, "sequence", rollappPacket.Packet.GetSequence(), "destination channel", rollappPacket.Packet.GetDestChannel(), "type", rollappPacket.Type)
 		// Update the packet status
-		im.keeper.UpdateRollappPacketStatus(ctx, rollappID, rollappPacket, types.RollappPacket_ACCEPTED)
-		// Call the onRecvPacket callback for each packet
-		ack := im.app.OnRecvPacket(ctx, *rollappPacket.Packet, rollappPacket.Relayer)
-		// Write the acknowledgement to the chain only if it is synchronous
-		if ack != nil {
-			_, chanCap, err := im.keeper.LookupModuleByChannel(ctx, rollappPacket.Packet.DestinationPort, rollappPacket.Packet.DestinationChannel)
+		rollappPacket := im.keeper.UpdateRollappPacketStatus(ctx, rollappPacket, types.RollappPacket_ACCEPTED)
+		// Call the relevant callback for each packet
+		switch rollappPacket.Type {
+		case types.RollappPacket_ON_RECV:
+			logger.Debug("Calling OnRecvPacket", "rollappID", rollappID, "sequence", rollappPacket.Packet.GetSequence(), "destination channel", rollappPacket.Packet.GetDestChannel())
+			wrappedFunc := func(ctx sdk.Context) error {
+				ack := im.app.OnRecvPacket(ctx, *rollappPacket.Packet, rollappPacket.Relayer)
+				if !ack.Success() {
+					return sdkerrors.Wrap(sdkerrors.ErrInvalidRequest, string(ack.Acknowledgement()))
+				}
+				// Write the acknowledgement to the chain only if it is synchronous
+				if ack != nil {
+					_, chanCap, err := im.keeper.LookupModuleByChannel(ctx, rollappPacket.Packet.DestinationPort, rollappPacket.Packet.DestinationChannel)
+					if err != nil {
+						return err
+					}
+					err = im.keeper.WriteAcknowledgement(ctx, chanCap, rollappPacket.Packet, ack)
+					if err != nil {
+						return err
+					}
+				}
+				return nil
+			}
+			err := osmoutils.ApplyFuncIfNoError(ctx, wrappedFunc)
 			if err != nil {
-				logger.Error("Error looking up module by channel", "rollappID", rollappID, "error", "sequence", rollappPacket.Packet.GetSequence(), "destination channel", rollappPacket.Packet.GetDestChannel(), "error", err.Error())
+				logger.Error("Error writing acknowledgement", "rollappID", rollappID, "sequence", rollappPacket.Packet.GetSequence(), "destination channel", rollappPacket.Packet.GetDestChannel(), "error", err.Error())
+				// Update the packet with the error
+				rollappPacket.Error = err.Error()
+				im.keeper.SetRollappPacket(ctx, rollappPacket)
 				continue
+
 			}
-			err = im.keeper.WriteAcknowledgement(ctx, chanCap, rollappPacket.Packet, ack)
+		case types.RollappPacket_ON_ACK:
+			logger.Debug("Calling OnAcknowledgementPacket", "rollappID", rollappID, "sequence", rollappPacket.Packet.GetSequence(), "destination channel", rollappPacket.Packet.GetDestChannel())
+			wrappedFunc := func(ctx sdk.Context) error {
+				err := im.app.OnAcknowledgementPacket(ctx, *rollappPacket.Packet, rollappPacket.Acknowledgement, rollappPacket.Relayer)
+				if err != nil {
+					return err
+				}
+				return nil
+			}
+			err := osmoutils.ApplyFuncIfNoError(ctx, wrappedFunc)
 			if err != nil {
-				logger.Error("Error writing acknowledgement", "rollappID", rollappID, "sequence", rollappPacket.Packet.GetSequence(), "destination channel", rollappPacket.Packet.GetDestChannel(), "error", err.Error())
+				logger.Error("Error calling OnAcknowledgementPacket", "rollappID", rollappID, "sequence", rollappPacket.Packet.GetSequence(), "destination channel", rollappPacket.Packet.GetDestChannel(), "error", err.Error())
+				rollappPacket.Error = err.Error()
+				im.keeper.SetRollappPacket(ctx, rollappPacket)
 				continue
 			}
-
+		case types.RollappPacket_ON_TIMEOUT:
+			logger.Debug("Calling OnTimeoutPacket", "rollappID", rollappID, "sequence", rollappPacket.Packet.GetSequence(), "destination channel", rollappPacket.Packet.GetDestChannel())
+			wrappedFunc := func(ctx sdk.Context) error {
+				err := im.app.OnTimeoutPacket(ctx, *rollappPacket.Packet, rollappPacket.Relayer)
+				if err != nil {
+					return err
+				}
+				return nil
+			}
+			err := osmoutils.ApplyFuncIfNoError(ctx, wrappedFunc)
+			if err != nil {
+				logger.Error("Error calling OnTimeoutPacket", "rollappID", rollappID, "sequence", rollappPacket.Packet.GetSequence(), "destination channel", rollappPacket.Packet.GetDestChannel(), "error", err.Error())
+				rollappPacket.Error = err.Error()
+				im.keeper.SetRollappPacket(ctx, rollappPacket)
+			}
+		default:
+			logger.Error("Unknown rollapp packet type", "rollappID", rollappID, "sequence", rollappPacket.Packet.GetSequence(), "destination channel", rollappPacket.Packet.GetDestChannel(), "type", rollappPacket.Type)
 		}
+
 	}
 }

x/delayedack/ibc_middleware.go

@@ -146,12 +146,14 @@ func (im IBCMiddleware) OnRecvPacket(
 
 	// Save the packet data to the store for later processing
 	rollappPacket := types.RollappPacket{
+		RollappId:   chainID,
 		Packet:      &packet,
 		Status:      types.RollappPacket_PENDING,
 		Relayer:     relayer,
 		ProofHeight: ibcClientLatestHeight.GetRevisionHeight(),
+		Type:        types.RollappPacket_ON_RECV,
 	}
-	im.keeper.SetRollappPacket(ctx, chainID, rollappPacket)
+	im.keeper.SetRollappPacket(ctx, rollappPacket)
 
 	return nil
 }
@@ -163,7 +165,65 @@ func (im IBCMiddleware) OnAcknowledgementPacket(
 	acknowledgement []byte,
 	relayer sdk.AccAddress,
 ) error {
-	return im.app.OnAcknowledgementPacket(ctx, packet, acknowledgement, relayer)
+	if !im.keeper.IsRollappsEnabled(ctx) {
+		return im.app.OnAcknowledgementPacket(ctx, packet, acknowledgement, relayer)
+	}
+	logger := ctx.Logger().With("module", "DelayedAckMiddleware")
+
+	// no-op if the packet is not a fungible token packet
+	var data transfertypes.FungibleTokenPacketData
+	if err := transfertypes.ModuleCdc.UnmarshalJSON(packet.GetData(), &data); err != nil {
+		return err
+	}
+
+	// Check if the packet is destined for a rollapp
+	chainID, err := im.keeper.ExtractChainIDFromChannel(ctx, packet.DestinationPort, packet.DestinationChannel)
+	if err != nil {
+		logger.Error("Failed to extract chain id from channel", "err", err)
+		return err
+	}
+
+	_, found := im.keeper.GetRollapp(ctx, chainID)
+	if !found {
+		logger.Debug("Skipping IBC transfer OnAcknowledgementPacket for non-rollapp chain")
+		return im.app.OnAcknowledgementPacket(ctx, packet, acknowledgement, relayer)
+	}
+
+	// Get the light client height at this block height as a proxy for the packet proof height
+	clientState, err := im.keeper.GetClientState(ctx, packet)
+	if err != nil {
+		return err
+	}
+
+	// TODO(omritoptix): Currently we use this height as the proofHeight as the real proofHeight from the ibc lower stack is not available.
+	// using this height is secured but may cause extra delay as at best it will be equal to the proof height (but could be higher).
+	ibcClientLatestHeight := clientState.GetLatestHeight()
+	finalizedHeight, err := im.keeper.GetRollappFinalizedHeight(ctx, chainID)
+	if err == nil && finalizedHeight >= ibcClientLatestHeight.GetRevisionHeight() {
+		logger.Debug("Skipping IBC transfer OnAcknowledgementPacket as the packet proof height is already finalized")
+		return im.app.OnAcknowledgementPacket(ctx, packet, acknowledgement, relayer)
+	}
+	// Run the underlying app's OnAcknowledgementPacket callback
+	// with cache context to avoid state changes and report the acknowledgement result.
+	// Only save the packet if the underlying app's callback succeeds.
+	cacheCtx, _ := ctx.CacheContext()
+	err = im.app.OnAcknowledgementPacket(cacheCtx, packet, acknowledgement, relayer)
+	if err != nil {
+		return err
+	}
+	// Save the packet data to the store for later processing
+	rollappPacket := types.RollappPacket{
+		RollappId:       chainID,
+		Packet:          &packet,
+		Acknowledgement: acknowledgement,
+		Status:          types.RollappPacket_PENDING,
+		Relayer:         relayer,
+		ProofHeight:     ibcClientLatestHeight.GetRevisionHeight(),
+		Type:            types.RollappPacket_ON_ACK,
+	}
+	im.keeper.SetRollappPacket(ctx, rollappPacket)
+
+	return nil
 }
 
 // OnTimeoutPacket implements the IBCMiddleware interface
@@ -172,8 +232,65 @@ func (im IBCMiddleware) OnTimeoutPacket(
 	packet channeltypes.Packet,
 	relayer sdk.AccAddress,
 ) error {
-	// call underlying callback
-	return im.app.OnTimeoutPacket(ctx, packet, relayer)
+	if !im.keeper.IsRollappsEnabled(ctx) {
+		return im.app.OnTimeoutPacket(ctx, packet, relayer)
+	}
+	logger := ctx.Logger().With("module", "DelayedAckMiddleware")
+
+	// no-op if the packet is not a fungible token packet
+	var data transfertypes.FungibleTokenPacketData
+	if err := transfertypes.ModuleCdc.UnmarshalJSON(packet.GetData(), &data); err != nil {
+		return err
+	}
+
+	// Check if the packet is destined for a rollapp
+	chainID, err := im.keeper.ExtractChainIDFromChannel(ctx, packet.DestinationPort, packet.DestinationChannel)
+	if err != nil {
+		logger.Error("Failed to extract chain id from channel", "err", err)
+		return err
+	}
+
+	_, found := im.keeper.GetRollapp(ctx, chainID)
+	if !found {
+		logger.Debug("Skipping IBC transfer OnTimeoutPacket for non-rollapp chain")
+		return im.app.OnTimeoutPacket(ctx, packet, relayer)
+	}
+
+	// Get the light client height at this block height as a proxy for the packet proof height
+	clientState, err := im.keeper.GetClientState(ctx, packet)
+	if err != nil {
+		return err
+	}
+
+	// TODO(omritoptix): Currently we use this height as the proofHeight as the real proofHeight from the ibc lower stack is not available.
+	// using this height is secured but may cause extra delay as at best it will be equal to the proof height (but could be higher).
+	ibcClientLatestHeight := clientState.GetLatestHeight()
+	finalizedHeight, err := im.keeper.GetRollappFinalizedHeight(ctx, chainID)
+	if err == nil && finalizedHeight >= ibcClientLatestHeight.GetRevisionHeight() {
+		logger.Debug("Skipping IBC transfer OnTimeoutPacket as the packet proof height is already finalized")
+		return im.app.OnTimeoutPacket(ctx, packet, relayer)
+	}
+
+	// Run the underlying app's OnTimeoutPacket callback
+	// with cache context to avoid state changes and report the timeout result.
+	// Only save the packet if the underlying app's callback succeeds.
+	cacheCtx, _ := ctx.CacheContext()
+	err = im.app.OnTimeoutPacket(cacheCtx, packet, relayer)
+	if err != nil {
+		return err
+	}
+	// Save the packet data to the store for later processing
+	rollappPacket := types.RollappPacket{
+		RollappId:   chainID,
+		Packet:      &packet,
+		Status:      types.RollappPacket_PENDING,
+		Relayer:     relayer,
+		ProofHeight: ibcClientLatestHeight.GetRevisionHeight(),
+		Type:        types.RollappPacket_ON_TIMEOUT,
+	}
+	im.keeper.SetRollappPacket(ctx, rollappPacket)
+
+	return nil
 }
 
 /* ------------------------------- ICS4Wrapper ------------------------------ */