Variable size for trampoline onion (#2810)

The size of trampoline onions was fixed at 400 bytes. We remove this limit, allowing larger trampoline onions when necessary.
The size of the trampoline onion is still constrained by the size of the standard onion (1300 bytes).

Co-authored-by: t-bast <[email protected]>

eclair-core/src/main/scala/fr/acinq/eclair/crypto/Sphinx.scala

@@ -223,6 +223,8 @@ object Sphinx extends Logging {
     nextPacket
   }
 
+  def payloadsTotalSize(payloads: Seq[ByteVector]): Int = payloads.map(_.length + MacLength).sum.toInt
+
   /**
    * Create an encrypted onion packet that contains payloads for all nodes in the list.
    *
@@ -235,7 +237,7 @@ object Sphinx extends Logging {
    *         the shared secrets (one per node) can be used to parse returned failure messages if needed.
    */
   def create(sessionKey: PrivateKey, packetPayloadLength: Int, publicKeys: Seq[PublicKey], payloads: Seq[ByteVector], associatedData: Option[ByteVector32]): Try[PacketAndSecrets] = Try {
-    require(payloads.map(_.length + MacLength).sum <= packetPayloadLength, s"packet per-hop payloads cannot exceed $packetPayloadLength bytes")
+    require(payloadsTotalSize(payloads) <= packetPayloadLength, s"packet per-hop payloads cannot exceed $packetPayloadLength bytes")
     val (ephemeralPublicKeys, sharedsecrets) = computeEphemeralPublicKeysAndSharedSecrets(sessionKey, publicKeys)
     val filler = generateFiller("rho", packetPayloadLength, sharedsecrets.dropRight(1), payloads.dropRight(1))
 

eclair-core/src/main/scala/fr/acinq/eclair/payment/PaymentPacket.scala

@@ -22,7 +22,7 @@ import fr.acinq.bitcoin.scalacompat.Crypto.{PrivateKey, PublicKey}
 import fr.acinq.eclair.channel.{CMD_ADD_HTLC, CMD_FAIL_HTLC, CannotExtractSharedSecret, Origin}
 import fr.acinq.eclair.crypto.Sphinx
 import fr.acinq.eclair.payment.send.Recipient
-import fr.acinq.eclair.router.Router.{BlindedHop, ChannelHop, Route}
+import fr.acinq.eclair.router.Router.{BlindedHop, Route}
 import fr.acinq.eclair.wire.protocol.PaymentOnion.{FinalPayload, IntermediatePayload, PerHopPayload}
 import fr.acinq.eclair.wire.protocol._
 import fr.acinq.eclair.{CltvExpiry, CltvExpiryDelta, Feature, Features, MilliSatoshi, ShortChannelId, TimestampMilli, UInt64, randomKey}
@@ -250,8 +250,12 @@ object OutgoingPaymentPacket {
   }
   // @formatter:on
 
-  /** Build an encrypted onion packet from onion payloads and node public keys. */
-  def buildOnion(packetPayloadLength: Int, payloads: Seq[NodePayload], associatedData: ByteVector32): Either[OutgoingPaymentError, Sphinx.PacketAndSecrets] = {
+  /**
+   * Build an encrypted onion packet from onion payloads and node public keys.
+   * If packetPayloadLength_opt is provided, the onion will be padded to the requested length.
+   * In that case, packetPayloadLength_opt must be greater than the actual onion's content.
+   */
+  def buildOnion(payloads: Seq[NodePayload], associatedData: ByteVector32, packetPayloadLength_opt: Option[Int]): Either[OutgoingPaymentError, Sphinx.PacketAndSecrets] = {
     val sessionKey = randomKey()
     val nodeIds = payloads.map(_.nodeId)
     val payloadsBin = payloads
@@ -260,6 +264,7 @@ object OutgoingPaymentPacket {
         case Attempt.Successful(bits) => bits.bytes
         case Attempt.Failure(cause) => return Left(CannotCreateOnion(cause.message))
       }
+    val packetPayloadLength = packetPayloadLength_opt.getOrElse(Sphinx.payloadsTotalSize(payloadsBin))
     Sphinx.create(sessionKey, packetPayloadLength, nodeIds, payloadsBin, Some(associatedData)) match {
       case Failure(f) => Left(CannotCreateOnion(f.getMessage))
       case Success(packet) => Right(packet)
@@ -297,7 +302,7 @@ object OutgoingPaymentPacket {
     for {
       paymentTmp <- recipient.buildPayloads(paymentHash, route)
       outgoing <- getOutgoingChannel(privateKey, paymentTmp, route)
-      onion <- buildOnion(PaymentOnionCodecs.paymentOnionPayloadLength, outgoing.payment.payloads, paymentHash) // BOLT 2 requires that associatedData == paymentHash
+      onion <- buildOnion(outgoing.payment.payloads, paymentHash, Some(PaymentOnionCodecs.paymentOnionPayloadLength)) // BOLT 2 requires that associatedData == paymentHash
     } yield {
       val cmd = CMD_ADD_HTLC(replyTo, outgoing.payment.amount, paymentHash, outgoing.payment.expiry, onion.packet, outgoing.nextBlinding_opt, Origin.Hot(replyTo, upstream), commit = true)
       OutgoingPaymentPacket(cmd, outgoing.shortChannelId, onion.sharedSecrets)

eclair-core/src/main/scala/fr/acinq/eclair/payment/send/Recipient.scala

@@ -24,7 +24,7 @@ import fr.acinq.eclair.payment.OutgoingPaymentPacket._
 import fr.acinq.eclair.payment.{Bolt11Invoice, Bolt12Invoice, OutgoingPaymentPacket, PaymentBlindedRoute}
 import fr.acinq.eclair.router.Router._
 import fr.acinq.eclair.wire.protocol.PaymentOnion.{FinalPayload, IntermediatePayload, OutgoingBlindedPerHopPayload}
-import fr.acinq.eclair.wire.protocol.{GenericTlv, OnionRoutingPacket, PaymentOnionCodecs}
+import fr.acinq.eclair.wire.protocol.{GenericTlv, OnionRoutingPacket}
 import fr.acinq.eclair.{CltvExpiry, Features, InvoiceFeature, MilliSatoshi, MilliSatoshiLong, ShortChannelId}
 import scodec.bits.ByteVector
 
@@ -230,14 +230,14 @@ case class ClearTrampolineRecipient(invoice: Bolt11Invoice,
       val finalPayload = NodePayload(nodeId, FinalPayload.Standard.createPayload(totalAmount, totalAmount, expiry, invoice.paymentSecret, invoice.paymentMetadata, customTlvs))
       val trampolinePayload = NodePayload(trampolineHop.nodeId, IntermediatePayload.NodeRelay.Standard(totalAmount, expiry, nodeId))
       val payloads = Seq(trampolinePayload, finalPayload)
-      OutgoingPaymentPacket.buildOnion(PaymentOnionCodecs.trampolineOnionPayloadLength, payloads, paymentHash)
+      OutgoingPaymentPacket.buildOnion(payloads, paymentHash, packetPayloadLength_opt = None)
     } else {
       // The recipient doesn't support trampoline: the trampoline node will convert the payment to a non-trampoline payment.
       // The final payload will thus never reach the recipient, so we create the smallest payload possible to avoid overflowing the trampoline onion size.
       val dummyFinalPayload = NodePayload(nodeId, IntermediatePayload.ChannelRelay.Standard(ShortChannelId(0), 0 msat, CltvExpiry(0)))
       val trampolinePayload = NodePayload(trampolineHop.nodeId, IntermediatePayload.NodeRelay.Standard.createNodeRelayToNonTrampolinePayload(totalAmount, totalAmount, expiry, nodeId, invoice))
       val payloads = Seq(trampolinePayload, dummyFinalPayload)
-      OutgoingPaymentPacket.buildOnion(PaymentOnionCodecs.trampolineOnionPayloadLength, payloads, paymentHash)
+      OutgoingPaymentPacket.buildOnion(payloads, paymentHash, packetPayloadLength_opt = None)
     }
   }
 }

eclair-core/src/main/scala/fr/acinq/eclair/wire/protocol/OnionRouting.scala

@@ -20,7 +20,7 @@ import fr.acinq.bitcoin.scalacompat.ByteVector32
 import fr.acinq.eclair.UInt64
 import fr.acinq.eclair.wire.protocol.CommonCodecs.bytes32
 import scodec.Codec
-import scodec.bits.ByteVector
+import scodec.bits.{BitVector, ByteVector}
 import scodec.codecs._
 
 /**
@@ -46,4 +46,18 @@ object OnionRoutingCodecs {
       ("onionPayload" | bytes(payloadLength)) ::
       ("hmac" | bytes32)).as[OnionRoutingPacket]
 
+  /**
+   * This codec encodes onion packets of variable sizes, and decodes the whole input byte stream into an onion packet.
+   * When decoding, the caller must ensure that they provide only the bytes that contain the onion packet.
+   */
+  val variableSizeOnionRoutingPacketCodec: Codec[OnionRoutingPacket] = (
+    ("version" | uint8) ::
+      ("publicKey" | bytes(33)) ::
+      ("onionPayload" | Codec(
+        // We simply encode the whole payload, nothing fancy here.
+        (payload: ByteVector) => bytes(payload.length.toInt).encode(payload),
+        // We stop 32 bytes before the end to avoid reading the hmac.
+        (bin: BitVector) => bytes((bin.length.toInt / 8) - 32).decode(bin))) ::
+      ("hmac" | bytes32)).as[OnionRoutingPacket]
+
 }

eclair-core/src/main/scala/fr/acinq/eclair/wire/protocol/PaymentOnion.scala

@@ -474,9 +474,7 @@ object PaymentOnionCodecs {
   import scodec.{Codec, DecodeResult, Decoder}
 
   val paymentOnionPayloadLength = 1300
-  val trampolineOnionPayloadLength = 400
   val paymentOnionPacketCodec: Codec[OnionRoutingPacket] = OnionRoutingCodecs.onionRoutingPacketCodec(paymentOnionPayloadLength)
-  val trampolineOnionPacketCodec: Codec[OnionRoutingPacket] = OnionRoutingCodecs.onionRoutingPacketCodec(trampolineOnionPayloadLength)
 
   /**
    * The 1.1 BOLT spec changed the payment onion frame format to use variable-length per-hop payloads.
@@ -509,7 +507,7 @@ object PaymentOnionCodecs {
 
   private val invoiceRoutingInfo: Codec[InvoiceRoutingInfo] = tlvField(list(listOfN(uint8, Bolt11Invoice.Codecs.extraHopCodec)))
 
-  private val trampolineOnion: Codec[TrampolineOnion] = tlvField(trampolineOnionPacketCodec)
+  private val trampolineOnion: Codec[TrampolineOnion] = tlvField(OnionRoutingCodecs.variableSizeOnionRoutingPacketCodec)
 
   private val keySend: Codec[KeySend] = tlvField(bytes32)
 

eclair-core/src/test/scala/fr/acinq/eclair/crypto/SphinxSpec.scala

@@ -595,7 +595,7 @@ object SphinxSpec {
     PaymentOnionCodecs.paymentOnionPacketCodec.encode(onion).require.toByteVector
 
   def serializeTrampolineOnion(onion: OnionRoutingPacket): ByteVector =
-    PaymentOnionCodecs.trampolineOnionPacketCodec.encode(onion).require.toByteVector
+    OnionRoutingCodecs.onionRoutingPacketCodec(onion.payload.length.toInt).encode(onion).require.toByteVector
 
   def createCustomLengthFailurePacket(failure: FailureMessage, sharedSecret: ByteVector32, length: Int): ByteVector = {
     val um = Sphinx.generateKey("um", sharedSecret)

eclair-core/src/test/scala/fr/acinq/eclair/payment/PaymentPacketSpec.scala

@@ -356,6 +356,7 @@ class PaymentPacketSpec extends AnyFunSuite with BeforeAndAfterAll {
     assert(outer_c.totalAmount == amount_bc)
     assert(outer_c.expiry == expiry_bc)
     assert(outer_c.paymentSecret != invoice.paymentSecret)
+    assert(outer_c.records.get[OnionPaymentPayloadTlv.TrampolineOnion].get.packet.payload.size < 400)
     assert(inner_c.amountToForward == finalAmount)
     assert(inner_c.totalAmount == finalAmount)
     assert(inner_c.outgoingCltv == finalExpiry)
@@ -382,21 +383,43 @@ class PaymentPacketSpec extends AnyFunSuite with BeforeAndAfterAll {
     assert(payload_e == FinalPayload.Standard(TlvStream(AmountToForward(finalAmount), OutgoingCltv(finalExpiry), PaymentData(invoice.paymentSecret, finalAmount), OnionPaymentPayloadTlv.PaymentMetadata(hex"010203"))))
   }
 
-  test("fail to build outgoing trampoline payment when too much invoice data is provided") {
-    val routingHintOverflow = List(List.fill(7)(Bolt11Invoice.ExtraHop(randomKey().publicKey, ShortChannelId(1), 10 msat, 100, CltvExpiryDelta(12))))
-    val invoice = Bolt11Invoice(Block.RegtestGenesisBlock.hash, Some(finalAmount), paymentHash, priv_e.privateKey, Left("#reckless"), CltvExpiryDelta(18), None, None, routingHintOverflow)
+  test("build outgoing trampoline payment with non-trampoline recipient (large invoice data)") {
+    // simple trampoline route to e where e doesn't support trampoline:
+    //             .----.
+    //            /      \
+    // a -> b -> c        e
+    // e provides many routing hints and a lot of payment metadata.
+    val routingHints = List(List.fill(7)(Bolt11Invoice.ExtraHop(randomKey().publicKey, ShortChannelId(1), 10 msat, 100, CltvExpiryDelta(12))))
+    val paymentMetadata = ByteVector.fromValidHex("2a" * 450)
+    val invoiceFeatures = Features[Bolt11Feature](VariableLengthOnion -> Mandatory, PaymentSecret -> Mandatory, BasicMultiPartPayment -> Optional)
+    val invoice = Bolt11Invoice(Block.RegtestGenesisBlock.hash, Some(finalAmount), paymentHash, priv_e.privateKey, Left("#reckless"), CltvExpiryDelta(18), extraHops = routingHints, features = invoiceFeatures, paymentMetadata = Some(paymentMetadata))
     val recipient = ClearTrampolineRecipient(invoice, finalAmount, finalExpiry, trampolineHop, randomBytes32())
-    val Left(failure) = buildOutgoingPayment(ActorRef.noSender, priv_a.privateKey, Upstream.Local(UUID.randomUUID()), paymentHash, Route(recipient.trampolineAmount, trampolineChannelHops, Some(trampolineHop)), recipient)
-    assert(failure.isInstanceOf[CannotCreateOnion])
-  }
+    val Right(payment) = buildOutgoingPayment(ActorRef.noSender, priv_a.privateKey, Upstream.Local(UUID.randomUUID()), paymentHash, Route(recipient.trampolineAmount, trampolineChannelHops, Some(trampolineHop)), recipient)
+    assert(payment.outgoingChannel == channelUpdate_ab.shortChannelId)
 
-  test("fail to build outgoing trampoline payment when too much payment metadata is provided") {
-    val paymentMetadata = ByteVector.fromValidHex("01" * 400)
-    val invoiceFeatures = Features[Bolt11Feature](VariableLengthOnion -> Mandatory, PaymentSecret -> Mandatory, BasicMultiPartPayment -> Optional, PaymentMetadata -> Optional, TrampolinePaymentPrototype -> Optional)
-    val invoice = Bolt11Invoice(Block.RegtestGenesisBlock.hash, Some(finalAmount), paymentHash, priv_e.privateKey, Left("Much payment very metadata"), CltvExpiryDelta(9), features = invoiceFeatures, paymentMetadata = Some(paymentMetadata))
-    val recipient = ClearTrampolineRecipient(invoice, finalAmount, finalExpiry, trampolineHop, randomBytes32())
-    val Left(failure) = buildOutgoingPayment(ActorRef.noSender, priv_a.privateKey, Upstream.Local(UUID.randomUUID()), paymentHash, Route(recipient.trampolineAmount, trampolineChannelHops, Some(trampolineHop)), recipient)
-    assert(failure.isInstanceOf[CannotCreateOnion])
+    val add_b = UpdateAddHtlc(randomBytes32(), 1, payment.cmd.amount, paymentHash, payment.cmd.cltvExpiry, payment.cmd.onion, None)
+    val Right(ChannelRelayPacket(_, _, packet_c)) = decrypt(add_b, priv_b.privateKey, Features.empty)
+
+    val add_c = UpdateAddHtlc(randomBytes32(), 2, amount_bc, paymentHash, expiry_bc, packet_c, None)
+    val Right(NodeRelayPacket(_, outer_c, inner_c, _)) = decrypt(add_c, priv_c.privateKey, Features.empty)
+    assert(outer_c.records.get[OnionPaymentPayloadTlv.TrampolineOnion].get.packet.payload.size > 800)
+    assert(inner_c.outgoingNodeId == e)
+    assert(inner_c.paymentMetadata.contains(paymentMetadata))
+    assert(inner_c.invoiceRoutingInfo.contains(routingHints))
+
+    // c forwards the trampoline payment to e through d.
+    val recipient_e = ClearRecipient(e, Features.empty, inner_c.amountToForward, inner_c.outgoingCltv, inner_c.paymentSecret.get, invoice.extraEdges, inner_c.paymentMetadata)
+    val Right(payment_e) = buildOutgoingPayment(ActorRef.noSender, priv_c.privateKey, Upstream.Trampoline(Seq(Upstream.ReceivedHtlc(add_c, TimestampMilli(1687345927000L)))), paymentHash, Route(inner_c.amountToForward, afterTrampolineChannelHops, None), recipient_e)
+    assert(payment_e.outgoingChannel == channelUpdate_cd.shortChannelId)
+    val add_d = UpdateAddHtlc(randomBytes32(), 3, payment_e.cmd.amount, paymentHash, payment_e.cmd.cltvExpiry, payment_e.cmd.onion, None)
+    val Right(ChannelRelayPacket(add_d2, payload_d, packet_e)) = decrypt(add_d, priv_d.privateKey, Features.empty)
+    assert(add_d2 == add_d)
+    assert(payload_d == IntermediatePayload.ChannelRelay.Standard(channelUpdate_de.shortChannelId, amount_de, expiry_de))
+
+    val add_e = UpdateAddHtlc(randomBytes32(), 4, amount_de, paymentHash, expiry_de, packet_e, None)
+    val Right(FinalPacket(add_e2, payload_e)) = decrypt(add_e, priv_e.privateKey, Features.empty)
+    assert(add_e2 == add_e)
+    assert(payload_e == FinalPayload.Standard(TlvStream(AmountToForward(finalAmount), OutgoingCltv(finalExpiry), PaymentData(invoice.paymentSecret, finalAmount), OnionPaymentPayloadTlv.PaymentMetadata(paymentMetadata))))
   }
 
   test("fail to build outgoing payment with invalid route") {

eclair-core/src/test/scala/fr/acinq/eclair/payment/relay/NodeRelayerSpec.scala

@@ -863,7 +863,7 @@ object NodeRelayerSpec {
 
   // This is the result of decrypting the incoming trampoline onion packet.
   // It should be forwarded to the next trampoline node.
-  val nextTrampolinePacket = OnionRoutingPacket(0, hex"02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619", randomBytes(PaymentOnionCodecs.trampolineOnionPayloadLength), randomBytes32())
+  val nextTrampolinePacket = OnionRoutingPacket(0, hex"02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619", randomBytes(400), randomBytes32())
 
   val outgoingAmount = 40_000_000 msat
   val outgoingExpiry = CltvExpiry(490000)

eclair-core/src/test/scala/fr/acinq/eclair/payment/relay/RelayerSpec.scala

@@ -25,6 +25,7 @@ import com.typesafe.config.ConfigFactory
 import fr.acinq.bitcoin.scalacompat.{Block, ByteVector32}
 import fr.acinq.eclair.FeatureSupport.{Mandatory, Optional}
 import fr.acinq.eclair.Features._
+import fr.acinq.eclair._
 import fr.acinq.eclair.channel._
 import fr.acinq.eclair.crypto.Sphinx
 import fr.acinq.eclair.payment.Bolt11Invoice
@@ -37,7 +38,6 @@ import fr.acinq.eclair.router.BaseRouterSpec.{blindedRouteFromHops, channelHopFr
 import fr.acinq.eclair.router.Router.{NodeHop, Route}
 import fr.acinq.eclair.wire.protocol.PaymentOnion.FinalPayload
 import fr.acinq.eclair.wire.protocol._
-import fr.acinq.eclair._
 import org.scalatest.concurrent.PatienceConfiguration
 import org.scalatest.funsuite.FixtureAnyFunSuiteLike
 import org.scalatest.{Outcome, Tag}
@@ -119,7 +119,7 @@ class RelayerSpec extends ScalaTestWithActorTestKit(ConfigFactory.load("applicat
     // We simulate a payment split between multiple trampoline routes.
     val totalAmount = finalAmount * 3
     val finalTrampolinePayload = NodePayload(b, FinalPayload.Standard.createPayload(finalAmount, totalAmount, finalExpiry, paymentSecret))
-    val Right(trampolineOnion) = buildOnion(PaymentOnionCodecs.trampolineOnionPayloadLength, Seq(finalTrampolinePayload), paymentHash)
+    val Right(trampolineOnion) = buildOnion(Seq(finalTrampolinePayload), paymentHash, None)
     val recipient = ClearRecipient(b, nodeParams.features.invoiceFeatures(), finalAmount, finalExpiry, randomBytes32(), nextTrampolineOnion_opt = Some(trampolineOnion.packet))
     val Right(payment) = buildOutgoingPayment(ActorRef.noSender, priv_a.privateKey, Upstream.Local(UUID.randomUUID()), paymentHash, Route(finalAmount, Seq(channelHopFromUpdate(priv_a.publicKey, b, channelUpdate_ab)), None), recipient)
     assert(payment.cmd.amount == finalAmount)