Tunnel-mode reception, and five object-handling fixes (compression, partitioning, transfer length)

include/AlcPacket.h

@@ -47,7 +47,8 @@ namespace LibFlute {
       *  @param close_object_flag Set the LCT Close Object flag (RFC 3451 clause 5.1, 'B' bit) on this packet
       */
       AlcPacket(uint64_t tsi, uint16_t toi, FecOti fec_oti, const std::vector<EncodingSymbol>& symbols, size_t max_size, uint32_t fdt_instance_id,
-                bool close_session_flag = false, bool close_object_flag = false);
+                bool close_session_flag = false, bool close_object_flag = false,
+                bool include_fti = false);
 
      /**
       *  Default destructor.

include/File.h

@@ -129,6 +129,16 @@ namespace LibFlute {
         _meta.content_type = fdt_entry.content_type;
         _meta.content_md5 = fdt_entry.content_md5;
         _meta.expires = fdt_entry.expires;
+        // The encoding has to come across too. Reception bootstrapped from EXT_FTI cannot know an
+        // object is content encoded, because EXT_FTI carries FEC parameters and not that; without
+        // this the object is delivered still compressed, with a Content-MD5 that will not match.
+        _meta.content_encoding = fdt_entry.content_encoding;
+        // And so does the content length. Bootstrapping sets it from EXT_FTI's transfer length,
+        // which is the same number only for an object carried without a content encoding. For an
+        // encoded one the FDT holds the decoded length and EXT_FTI the encoded one, so keeping the
+        // bootstrap value makes the post-decode length check compare against the wrong figure.
+        // fec_oti is still left alone: its transfer length is what the reassembly is keyed on.
+        _meta.content_length = fdt_entry.content_length;
       };
 
      /**

include/Receiver.h

@@ -18,9 +18,12 @@
 #include <boost/bind/bind.hpp>
 #include <atomic>
 #include <memory>
+#include <optional>
 #include <string>
 #include <map>
 #include <mutex>
+#include <set>
+#include <vector>
 #include "File.h"
 #include "FileDeliveryTable.h"
 
@@ -48,6 +51,44 @@ namespace LibFlute {
       *  @param toi The TOI the packet carrying the flag(s) was for
       */
       typedef std::function<void(bool session_closed, bool object_closed, uint32_t toi)> close_notification_callback_t;
+
+     /**
+      *  Definition of the caller-supplied hook that locates the ALC/LCT payload inside a
+      *  tunnelled datagram.
+      *
+      *  https://github.com/5G-MAG/rt-libflute/issues/66 : the Receiver-side counterpart to
+      *  Transmitter's existing udp_tunnel_address() support (see its _tunnel_endpoint /
+      *  create_ip_hdr() / create_udp_pkt()). Per the issue discussion: the library has no
+      *  business knowing about any particular encapsulation format -- that is entirely the
+      *  controlling application's concern ("a better design pattern would be for the
+      *  controlling application to pass in a 'helper' function that the library invokes to
+      *  do application-specific mangling of packets before the generic code in the library
+      *  starts processing the ALC/LCT payload", rjb1000). This is that helper's signature.
+      *
+      *  This mirrors the de-tunnelling logic that already exists, hand-written, in
+      *  tests/test_end_to_end.cpp's run_tunnel_bridge() (added alongside Transmitter's own
+      *  tunnel mode) -- a std::thread there receives on a plain UDP socket, manually parses
+      *  a hand-built inner IPv4+UDP header out of the payload (no GTP-U or other framing;
+      *  Transmitter's tunnel mode wraps the ALC packet in exactly this and nothing else),
+      *  and forwards just the FLUTE bytes onward over loopback to a receiver that has no
+      *  tunnel-awareness at all. This issue asks for that same logic to move inside Receiver
+      *  proper, as a caller-supplied, protocol-agnostic hook rather than a hardcoded parser
+      *  baked into the library, so any encapsulation a given deployment actually needs
+      *  (matching Transmitter's own wrapper, real GTP-U, or anything else) is expressed
+      *  purely by what modifier the caller passes in.
+      *
+      *  @param payload The whole datagram as received on the tunnel socket. The callback may
+      *         freely inspect and/or edit its contents in place (e.g. to decrypt a payload
+      *         that arrives encrypted under the encapsulation, not just to locate it).
+      *  @return The byte offset within (the, possibly now-modified) @p payload at which the
+      *          ALC/LCT payload begins. A return value >= payload.size() (e.g. SIZE_MAX)
+      *          tells the Receiver to silently discard this datagram without attempting ALC
+      *          parsing -- there is no separate bool/optional discard signal; encoding
+      *          "nothing usable here" as "there are no bytes left to read" keeps this one
+      *          consistent, easy-to-satisfy contract rather than two.
+      */
+      typedef std::function<size_t(std::vector<uint8_t>& payload)> packet_modifier_t;
+
      /**
       *  Default constructor.
       *
@@ -58,11 +99,45 @@ namespace LibFlute {
       *  @param io_context Boost io_context to run the socket operations in (must be provided by the caller)
       *  @param source_address If non-empty, join as source-specific multicast (SSM, IPv4
       *         only) admitting only packets from this source -- otherwise ASM (any-source).
+      *         Unrelated to the tunnel parameters below; this is the pre-existing plain
+      *         multicast join, unchanged.
+      *  @param tunnel_address If given, ALSO bind a plain unicast UDP socket to this local
+      *         endpoint and accept tunnelled datagrams there, in addition to the normal
+      *         multicast join above. Motivating cases (see the issue): a deployment that
+      *         needs to receive FLUTE content arriving encapsulated (e.g. GTP-U) rather than
+      *         as plain IP multicast, or one where the platform's local multicast delivery
+      *         isn't available on the path content actually arrives over at all (confirmed
+      *         live: a datagram written to a software TUN device, as a UE simulator's own
+      *         decapsulated-content path does, never reaches a socket joined to its
+      *         destination multicast group on that interface -- the platform's multicast
+      *         delivery never triggers for it at all -- even though the identical datagram
+      *         delivers correctly on a real network interface). The two paths are
+      *         independent and both feed the same session state, so either one arriving is
+      *         enough; this is deliberately not an either/or choice like Transmitter's
+      *         tunnel mode, since a Receiver has no way to know in advance which path will
+      *         actually work in a given deployment.
+      *  @param tunnel_source If given, only accept tunnel datagrams whose UDP source address
+      *         matches this value -- the tunnel-socket equivalent of @p source_address's SSM
+      *         admit-only-this-source semantics, since the tunnel socket itself is a plain
+      *         unicast bind with no multicast-layer source filtering of its own. This is
+      *         the "extra address checking" the library itself does, on top of whatever
+      *         @p packet_modifier does -- source-address admission is a generic,
+      *         encapsulation-agnostic concept the library can reasonably own, unlike parsing
+      *         any particular header format.
+      *  @param packet_modifier Required whenever tunnel_address is set (ignored otherwise,
+      *         and if omitted while tunnel_address is set, every tunnel datagram is
+      *         discarded -- silently failing open would be far worse than silently
+      *         discarding). See packet_modifier_t; there is no default implementation, since
+      *         any default would itself bake an assumption about the tunnel's wire format
+      *         into the library, exactly what this issue asks not to do.
       */
       Receiver( const std::string& iface, const std::string& address,
           short port, uint64_t tsi,
           boost::asio::io_context& io_context,
-          const std::string& source_address = "");
+          const std::string& source_address = "",
+          const std::optional<boost::asio::ip::udp::endpoint>& tunnel_address = std::nullopt,
+          const std::optional<boost::asio::ip::address>& tunnel_source = std::nullopt,
+          const std::optional<packet_modifier_t>& packet_modifier = std::nullopt);
 
      /**
       *  Destructor. Marks the receiver as no longer alive so that any async_receive_from
@@ -120,9 +195,28 @@ namespace LibFlute {
       void handle_receive_from(const boost::system::error_code& error,
           size_t bytes_recvd);
       void arm_receive();
+      // The actual ALC/FLUTE processing, shared by both the normal multicast-socket path
+      // (handle_receive_from(), data already at offset 0 in _data) and the tunnel path
+      // (handle_tunnel_receive_from(), data at whatever offset packet_modifier_t returns
+      // inside _tunnel_data) -- see packet_modifier_t's comment in the header for why these
+      // are two independent, simultaneously-armed receive loops rather than a single one.
+      void process_alc_datagram(char* data, size_t len);
+      void handle_tunnel_receive_from(const boost::system::error_code& error,
+          size_t bytes_recvd);
+      void arm_tunnel_receive();
       boost::asio::ip::udp::socket _socket;
       boost::asio::ip::udp::endpoint _sender_endpoint;
 
+      std::unique_ptr<boost::asio::ip::udp::socket> _tunnel_socket;
+      boost::asio::ip::udp::endpoint _tunnel_sender_endpoint;
+      std::optional<boost::asio::ip::address> _tunnel_source;
+      packet_modifier_t _packet_modifier;
+      // Sized like _data (see max_length below) but as a resizable buffer rather than a
+      // fixed char array: packet_modifier_t takes a std::vector<uint8_t>& so a caller-supplied
+      // modifier can shrink/grow the datagram in place (e.g. after decrypting a payload that
+      // changes size), not just report where to start reading a fixed buffer.
+      std::vector<uint8_t> _tunnel_data;
+
       // Must hold the largest UDP datagram that can actually arrive: with the
       // Compact No-Code FEC scheme, a packet is a 4-byte SBN+ID header plus one
       // full encoding symbol, and FEC-OTI-Encoding-Symbol-Length is a per-session

src/AlcPacket.cpp

@@ -240,7 +240,8 @@ LibFlute::AlcPacket::AlcPacket(char* data, size_t len)
 }
 
 LibFlute::AlcPacket::AlcPacket(uint64_t tsi, uint16_t toi, LibFlute::FecOti fec_oti, const std::vector<LibFlute::EncodingSymbol>& symbols, size_t max_encoding_symbol_size, uint32_t fdt_instance_id,
-                                bool close_session_flag, bool close_object_flag)
+                                bool close_session_flag, bool close_object_flag,
+                                bool include_fti)
   : _fec_oti(fec_oti)
 {
   // TSI width: this wire scheme always carries a 16-bit half-word component (half_word_flag=1,
@@ -258,8 +259,10 @@ LibFlute::AlcPacket::AlcPacket(uint64_t tsi, uint16_t toi, LibFlute::FecOti fec_
   if (wide_tsi) {
     lct_header_len += 1;
   }
-  if (toi == 0) { // Add extensions for FDT
+  if (toi == 0) { // EXT_FDT (one word) plus EXT_FTI (four words)
     lct_header_len += 5;
+  } else if (include_fti) { // EXT_FTI only, four words
+    lct_header_len += 4;
   }
 
   auto max_packet_length = max_encoding_symbol_size +
@@ -295,14 +298,27 @@ LibFlute::AlcPacket::AlcPacket(uint64_t tsi, uint16_t toi, LibFlute::FecOti fec_
   *((uint16_t*)hdr_ptr) = htons(toi);
   hdr_ptr += 2;
 
-  if (toi == 0) { // Add extensions for FDT
+  if (toi == 0) { // EXT_FDT describes the FDT instance and belongs only on the FDT itself
     *((uint8_t*)hdr_ptr) = EXT_FDT;
     hdr_ptr += 1;
     *((uint8_t*)hdr_ptr) = 1 << 4 | (fdt_instance_id & 0x000F0000) >> 16;
     hdr_ptr += 1;
     *((uint16_t*)hdr_ptr) = htons(fdt_instance_id & 0x0000FFFF);
     hdr_ptr += 2;
+  }
 
+  /* EXT_FTI goes on the FDT always, and on a content object when asked for. The caller asks when
+     the FDT cannot carry the object's transfer length, which under the MBMS Download Profile is any
+     content-encoded object: TS 26.346 V18.2.0 clause L.4.4 forbids Transfer-Length in the FDT and
+     RFC 3926 clause 3.4.2 only lets Content-Length stand in for an unencoded object, so in band is
+     the only route left and RFC 3926 clause 5 obliges every receiver to support it.
+
+     This is a deliberate departure from a "should", taken because the alternative departs from a
+     "shall not". TS 26.346 V18.2.0 clause L.4.7: "FEC Object Transmission Information in FLUTE
+     packets which carry symbols of content files should be conveyed by the FEC-OTI parameters in
+     the FDT". The FDT still carries every FEC-OTI parameter the profile permits; what it cannot
+     carry, and what travels here instead, is the transfer length alone. */
+  if (toi == 0 || include_fti) {
     *((uint8_t*)hdr_ptr) = EXT_FTI;
     hdr_ptr += 1;
     *((uint8_t*)hdr_ptr) = 4; // HEL

src/File.cpp

@@ -206,7 +206,19 @@ auto File::check_file_completion() -> void
 
 auto File::calculate_partitioning() -> void
 {
-  // Calculate source block partitioning (RFC5052 9.1) 
+  /* Both divisors come from the FEC OTI and both are used as denominators below. A
+     default-constructed FecOti leaves them 0, which makes the first division produce inf, the
+     block count inf, and the block-creation loop below effectively unbounded: the object hangs
+     rather than reporting anything. Reachable through the public File constructors, which accept
+     a FecOti without inspecting it. Refused loudly instead, per RULES.md rule 12.
+     `code-derived, no spec claim`. */
+  if (_meta.fec_oti.encoding_symbol_length == 0 || _meta.fec_oti.max_source_block_length == 0) {
+    throw std::runtime_error(
+        "FEC OTI is unusable for partitioning: encoding_symbol_length and "
+        "max_source_block_length must both be non-zero");
+  }
+
+  // Calculate source block partitioning (RFC5052 9.1)
   _nof_source_symbols = ceil((double)_meta.fec_oti.transfer_length / (double)_meta.fec_oti.encoding_symbol_length);
   _nof_source_blocks = ceil((double)_nof_source_symbols / (double)_meta.fec_oti.max_source_block_length);
   _large_source_block_length = ceil((double)_nof_source_symbols / (double)_nof_source_blocks);
@@ -297,7 +309,19 @@ auto File::encode() -> void
       };
       spdlog::debug("Compressing contents with {}", _meta.content_encoding);
 
-      if (deflateInit2(&zs, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 | 16, 8, Z_DEFAULT_STRATEGY) == Z_OK) {
+      /* Select the framing from the encoding, matching inflateInit2 below. windowBits 15 is the
+         zlib wrapper, and adding 16 selects gzip instead. Hardcoding gzip here meant a file
+         declared as "deflate" was written with gzip framing, so this library could not read back
+         what it had just written, and the framing did not match what the declared encoding means.
+
+         RFC 9110 clause 8.4.1.2: "The "deflate" coding is a "zlib" data format [RFC1950]
+         containing a "deflate" compressed data stream [RFC1951] that uses a combination of the
+         Lempel-Ziv (LZ77) compression algorithm and Huffman coding."
+
+         General FLUTE only in practice: the MBMS Download Profile permits no encoding other than
+         gzip, so a 3GPP session never takes the deflate branch. */
+      const int window_bits = 15 | ((_meta.content_encoding == "gzip") ? 16 : 0);
+      if (deflateInit2(&zs, Z_DEFAULT_COMPRESSION, Z_DEFLATED, window_bits, 8, Z_DEFAULT_STRATEGY) == Z_OK) {
         _buffer = nullptr;
         auto zstate = deflate(&zs, Z_FINISH);
         size_t last_out = 0;
@@ -320,8 +344,14 @@ auto File::encode() -> void
           }
           _meta.fec_oti.transfer_length = zs.total_out;
         } else {
-          spdlog::error("Error compressing file {}: {}", _meta.toi, zs.msg);
-          throw zs.msg;
+          /* zs.msg is NULL for Z_STREAM_ERROR, so the previous form formatted a null char* through
+             spdlog and then threw it. Throwing a raw char* also meant only catch(const char*)
+             could handle it, and dereferencing the caught null would crash the handler. */
+          const char *zmsg = zs.msg ? zs.msg : "no zlib message";
+          spdlog::error("Error compressing file {}: {} (zlib status {})", _meta.toi, zmsg, zstate);
+          deflateEnd(&zs);
+          if (own_decomp) free(decomp_buffer);
+          throw std::runtime_error(std::string("Failed to compress file: ") + zmsg);
         }
         deflateEnd(&zs);
 
@@ -354,7 +384,15 @@ auto File::decode() -> void
 
       inflateInit2(&zs, 15 | ((_meta.content_encoding == "gzip")?16:0));
       _buffer = nullptr;
-      auto zstate = inflate(&zs, Z_FINISH);
+      /* Z_NO_FLUSH, not Z_FINISH. Z_FINISH promises inflate that the output buffer can hold the
+         whole result; with the 16384-byte staging buffer below that is only true for small
+         objects, and for anything larger inflate returns Z_BUF_ERROR with zs.msg left NULL. The
+         loop below continues on Z_OK, so it ran zero times and control fell straight into the
+         error branch, which then formatted that NULL pointer. Reproduced standalone: a 100,000
+         byte object returned Z_BUF_ERROR immediately with total_out stuck at 16384, and the same
+         input with Z_NO_FLUSH reached Z_STREAM_END in six iterations with all 100,000 bytes.
+         `code-derived, no spec claim`. */
+      auto zstate = inflate(&zs, Z_NO_FLUSH);
       size_t last_out = 0;
       while (zstate == Z_OK) {
         spdlog::debug("Part decompressed: {} bytes", 16384-zs.avail_out);
@@ -364,7 +402,7 @@ auto File::decode() -> void
         _own_buffer = true;
 	zs.avail_out = 16384;
         zs.next_out = decomp_buffer.get();
-	zstate = inflate(&zs, Z_FINISH);
+	zstate = inflate(&zs, Z_NO_FLUSH);
       }
       if (zstate==Z_STREAM_END) {
 	if (last_out != zs.total_out) {
@@ -379,8 +417,13 @@ auto File::decode() -> void
           spdlog::error("Decompressed length does not match expected Content-Length ({} != {})", _meta.content_length, zs.total_out);
         }
       } else {
-        spdlog::error("Error decompressing file {}: {}", _meta.toi, zs.msg);
-	throw zs.msg;
+        /* zs.msg is NULL for several zlib statuses, so the previous form formatted a null
+           char* through spdlog and then threw it, which only catch(const char*) could take. */
+        const char *zmsg = zs.msg ? zs.msg : "no zlib message";
+        spdlog::error("Error decompressing file {}: {} (zlib status {})", _meta.toi, zmsg, zstate);
+        inflateEnd(&zs);
+        if (own_comp) free(comp_buffer);
+        throw std::runtime_error(std::string("Failed to decompress file: ") + zmsg);
       }
 
       if (own_comp) free(comp_buffer);