issue #4052: let http remux support filter h264 nalu SEI.

trunk/conf/full.conf

@@ -1530,6 +1530,15 @@ vhost http.remux.srs.com {
         # Overwrite by env SRS_VHOST_HTTP_REMUX_GUESS_HAS_AV for all vhosts.
         # Default: on
         guess_has_av on;
+
+        # Whether to keep the h.264 SEI type NALU packet.
+        # DJI drone M30T will send many SEI type NALU packet, while iOS hardware decoder (Video Toolbox)
+        # dislike to feed it so many SEI NALU between NonIDR and IDR NALU packets.
+        # @see https://github.com/ossrs/srs/issues/4052
+        # Overwrite by env SRS_VHOST_HTTP_REMUX_KEEP_AVC_NALU_SEI for all vhosts.
+        # Default: on
+        keep_avc_nalu_sei on;
+
         # the stream mount for rtmp to remux to live streaming.
         # typical mount to [vhost]/[app]/[stream].flv
         # the variables:

trunk/src/app/srs_app_config.cpp

@@ -2646,7 +2646,8 @@ srs_error_t SrsConfig::check_normal_config()
                 for (int j = 0; j < (int)conf->directives.size(); j++) {
                     string m = conf->at(j)->name;
                     if (m != "enabled" && m != "mount" && m != "fast_cache" && m != "drop_if_not_match"
-                        && m != "has_audio" && m != "has_video" && m != "guess_has_av") {
+                        && m != "has_audio" && m != "has_video" && m != "guess_has_av"
+                        && m != "keep_avc_nalu_sei") {
                         return srs_error_new(ERROR_SYSTEM_CONFIG_INVALID, "illegal vhost.http_remux.%s of %s", m.c_str(), vhost->arg0().c_str());
                     }
                 }
@@ -8666,6 +8667,30 @@ bool SrsConfig::get_vhost_http_remux_guess_has_av(string vhost)
     return SRS_CONF_PERFER_TRUE(conf->arg0());
 }
 
+bool SrsConfig::get_vhost_http_remux_keep_avc_nalu_sei(string vhost)
+{
+    SRS_OVERWRITE_BY_ENV_BOOL2("srs.vhost.http_remux.keep_avc_nalu_sei"); // SRS_VHOST_HTTP_REMUX_KEEP_AVC_NALU_SEI
+
+    static bool DEFAULT = true;
+
+    SrsConfDirective* conf = get_vhost(vhost);
+    if (!conf) {
+        return DEFAULT;
+    }
+
+    conf = conf->get("http_remux");
+    if (!conf) {
+        return DEFAULT;
+    }
+
+    conf = conf->get("keep_avc_nalu_sei");
+    if (!conf || conf->arg0().empty()) {
+        return DEFAULT;
+    }
+
+    return SRS_CONF_PERFER_TRUE(conf->arg0());
+}
+
 string SrsConfig::get_vhost_http_remux_mount(string vhost)
 {
     SRS_OVERWRITE_BY_ENV_STRING("srs.vhost.http_remux.mount"); // SRS_VHOST_HTTP_REMUX_MOUNT

trunk/src/app/srs_app_config.hpp

@@ -1098,6 +1098,7 @@ class SrsConfig
     bool get_vhost_http_remux_has_video(std::string vhost);
     // Whether guessing stream about audio or video track
     bool get_vhost_http_remux_guess_has_av(std::string vhost);
+    bool get_vhost_http_remux_keep_avc_nalu_sei(std::string vhost);
     // Get the http flv live stream mount point for vhost.
     // used to generate the flv stream mount path.
     virtual std::string get_vhost_http_remux_mount(std::string vhost);

trunk/src/app/srs_app_http_stream.cpp

@@ -19,6 +19,8 @@ using namespace std;
 
 #include <srs_protocol_stream.hpp>
 #include <srs_protocol_utility.hpp>
+#include <srs_protocol_format.hpp>
+#include <srs_kernel_codec.hpp>
 #include <srs_kernel_log.hpp>
 #include <srs_kernel_error.hpp>
 #include <srs_app_st.hpp>
@@ -630,6 +632,7 @@ srs_error_t SrsLiveStream::do_serve_http(ISrsHttpResponseWriter* w, ISrsHttpMess
     bool has_audio = _srs_config->get_vhost_http_remux_has_audio(req->vhost);
     bool has_video = _srs_config->get_vhost_http_remux_has_video(req->vhost);
     bool guess_has_av = _srs_config->get_vhost_http_remux_guess_has_av(req->vhost);
+    bool keep_avc_nalu_sei = _srs_config->get_vhost_http_remux_keep_avc_nalu_sei(req->vhost);
 
     if (srs_string_ends_with(entry->pattern, ".flv")) {
         w->header()->set_content_type("video/x-flv");
@@ -719,6 +722,9 @@ srs_error_t SrsLiveStream::do_serve_http(ISrsHttpResponseWriter* w, ISrsHttpMess
         entry->pattern.c_str(), enc_desc.c_str(), srsu2msi(mw_sleep), enc->has_cache(), msgs.max, drop_if_not_match,
         has_audio, has_video, guess_has_av);
 
+    SrsRtmpFormat format;
+    format.initialize();
+
     // TODO: free and erase the disabled entry after all related connections is closed.
     // TODO: FXIME: Support timeout for player, quit infinite-loop.
     while (entry->enabled) {
@@ -748,7 +754,15 @@ srs_error_t SrsLiveStream::do_serve_http(ISrsHttpResponseWriter* w, ISrsHttpMess
             srs_trace("-> " SRS_CONSTS_LOG_HTTP_STREAM " http: got %d msgs, age=%d, min=%d, mw=%d",
                 count, pprint->age(), SRS_PERF_MW_MIN_MSGS, srsu2msi(mw_sleep));
         }
-
+
+        // Drop h.264 flv video tags with NALU SEI here to fix http-flv play error in safari mac.
+        // @see https://github.com/ossrs/srs/issues/4052
+        if (!keep_avc_nalu_sei) {
+            if ((err = filter_avc_nalu_sei(format, msgs.msgs, count)) != srs_success) {
+                return srs_error_wrap(err, "filter avc/h264 nalu sei");
+            }
+        }
+
         // sendout all messages.
         if (ffe) {
             err = ffe->write_tags(msgs.msgs, count);
@@ -763,7 +777,7 @@ srs_error_t SrsLiveStream::do_serve_http(ISrsHttpResponseWriter* w, ISrsHttpMess
             SrsSharedPtrMessage* msg = msgs.msgs[i];
             srs_freep(msg);
         }
-        
+
         // check send error code.
         if (err != srs_success) {
             return srs_error_wrap(err, "send messages");
@@ -873,6 +887,186 @@ srs_error_t SrsLiveStream::streaming_send_messages(ISrsBufferEncoder* enc, SrsSh
     return err;
 }
 
+srs_error_t SrsLiveStream::filter_avc_nalu_sei(SrsRtmpFormat& format, SrsSharedPtrMessage** msgs, int& count) {
+    srs_error_t err = srs_success;
+    bool has_sei = false;
+    if ((err = has_avc_nalu_sei(format, msgs, count, has_sei)) != srs_success) {
+        return srs_error_wrap(err, "check avc nalu sei");
+    }
+
+    if (!has_sei) {
+        return err;
+    }
+
+    std::vector<SrsSharedPtrMessage*> remuxed_msgs;
+    for (size_t i = 0; i < count; i++) {
+        SrsSharedPtrMessage* msg = msgs[i];
+        if (msg->is_video()) {
+            if ((err = format.on_video(msg)) != srs_success) {
+                return srs_error_wrap(err, "format consume video");
+            }
+
+            if (format.vcodec->id == SrsVideoCodecIdAVC &&
+                format.video->avc_packet_type == SrsVideoAvcFrameTraitNALU) {
+                std::vector<int> sei_idxes;
+                // found which nalu is SEI, store its index inorder to sei_idxes;
+                for (size_t j = 0; j < format.video->nb_samples; ++j) {
+                    SrsSample* sample = &format.video->samples[j];
+
+                    SrsAvcNaluType avc_nalu_type = SrsAvcNaluTypeReserved;
+                    if ((err = SrsVideoFrame::parse_avc_nalu_type(sample, avc_nalu_type)) != srs_success) {
+                        return srs_error_wrap(err, "parse avc nalu_type");
+                    }
+                    if (avc_nalu_type == SrsAvcNaluTypeSEI) {
+                        sei_idxes.push_back(j);
+                    }
+                }
+
+                if (sei_idxes.size() == 0) { // no sei nalu, keep this msg
+                    remuxed_msgs.push_back(msg->copy());
+                    continue;
+                } else if (sei_idxes.size() == format.video->nb_samples) {
+                    // all the nalu is SEI type, remove this msg
+                    continue;
+                } else {
+                    // part of nalu is SEI, do remux this msg.
+                    int nalus_count = format.video->nb_samples - sei_idxes.size();
+
+                    SrsSample nalus[nalus_count];
+                    size_t p = 0;
+                    size_t r = 0;
+                    for (size_t j = 0; j < format.video->nb_samples; j++) {
+                        if (sei_idxes[r] == j) {
+                            r++;
+                        } else {
+                            SrsSample* s = &format.video->samples[j];
+                            nalus[p].size = s->size;
+                            nalus[p].bytes = s->bytes;
+                            p++;
+                        }
+                    }
+
+                    SrsSharedPtrMessage* m = NULL;
+                    if ((err = remux_avc_nalus(format, msg, &m, nalus, nalus_count))
+                        != srs_success) {
+                        return srs_error_wrap(err, "remux avc nalus");
+                    }
+
+                    remuxed_msgs.push_back(m);
+                    continue;
+                }
+            }
+        }
+
+        remuxed_msgs.push_back(msg->copy());
+    }
+
+    // free the messages.
+    for (size_t i = 0; i < count; i++) {
+        SrsSharedPtrMessage* msg = msgs[i];
+        srs_freep(msg);
+    }
+
+    // copy remuxed_msgs -> msgs
+    count = remuxed_msgs.size();
+    SrsSharedPtrMessage** pmsgs = remuxed_msgs.data();
+    memcpy(msgs, pmsgs, count * sizeof(SrsSharedPtrMessage*));
+    return err;
+}
+
+srs_error_t SrsLiveStream::has_avc_nalu_sei(SrsRtmpFormat& format, SrsSharedPtrMessage** msgs, int& count,
+                                            bool& has_sei) {
+    srs_error_t err = srs_success;
+    has_sei = false;
+
+    for (int i = 0; i < count; i++) {
+        SrsSharedPtrMessage* msg = msgs[i];
+        if (msg->is_video()) {
+            if ((err = format.on_video(msg)) != srs_success) {
+                srs_error("has_sei: format.on_video error");
+                return srs_error_wrap(err, "format consume video");
+            }
+
+            if (format.vcodec->id == SrsVideoCodecIdAVC &&
+                format.video->avc_packet_type == SrsVideoAvcFrameTraitNALU) {
+
+                for (size_t i = 0; i < format.video->nb_samples; ++i) {
+                    SrsSample* sample = &format.video->samples[i];
+
+                    SrsAvcNaluType avc_nalu_type = SrsAvcNaluTypeReserved;
+                    if ((err = SrsVideoFrame::parse_avc_nalu_type(sample, avc_nalu_type)) != srs_success) {
+                        return srs_error_wrap(err, "parse avc nalu_type");
+                    }
+
+                    if (avc_nalu_type == SrsAvcNaluTypeSEI) {
+                        has_sei = true;
+                        return err;
+                    } 
+                }
+            }
+        }
+    }
+
+    return err;
+}
+
+srs_error_t SrsLiveStream::remux_avc_nalus(const SrsRtmpFormat& format,
+                                           const SrsSharedPtrMessage* msg,
+                                           SrsSharedPtrMessage** output_msg,
+                                           SrsSample* nalus,
+                                           size_t nalu_count)
+{
+    srs_error_t err = srs_success;
+
+    if (!msg->is_video()) {
+        return srs_error_new(ERROR_RTMP_MESSAGE_CREATE, "msg is not video");
+    }
+    // normal flv video tag header 5 bytes; 
+    // extened(enhanced) flv video tag header 8 bytes;
+    // but h264 use normal flv video tag header format;
+    const int video_tag_header_size = 5;
+    size_t nalu_header_size = 3;
+    if (format.vcodec->payload_format == SrsAvcPayloadFormatIbmf) {
+        nalu_header_size = format.vcodec->NAL_unit_length + 1;
+    } else if (format.vcodec->payload_format == SrsAvcPayloadFormatAnnexb) {
+        nalu_header_size = 3; // 00 00 01
+    } else {
+        return srs_error_new(ERROR_RTMP_MESSAGE_CREATE,
+                             "unkown nalu format type %d",
+                             format.vcodec->payload_format);
+    }
+
+    // calculate new video tag size;
+    size_t payload_size = video_tag_header_size;
+
+    for (size_t i = 0; i < nalu_count; i++) {
+        payload_size += nalus[i].size + nalu_header_size;
+    }
+
+    srs_assert(payload_size <= msg->size);
+
+    SrsSharedPtrMessage* m = new SrsSharedPtrMessage();
+    char* payload = (char*)malloc(payload_size);
+    char* payload_p = payload;
+    // memcpy flv header
+    memcpy(payload_p, msg->payload, video_tag_header_size);
+    payload_p += video_tag_header_size;
+    // memcpy nalus
+    for (size_t i = 0; i < nalu_count; i++) {
+        int n = nalus[i].size + nalu_header_size;
+        memcpy(payload_p, nalus[i].bytes - nalu_header_size, n);
+        payload_p += n;
+    }
+
+    if ((err = m->create(msg, payload, payload_size)) != srs_success) {
+        return srs_error_wrap(err, "create SrsSharedPtrMessage error.");
+    }
+
+    *output_msg = m;
+
+    return err;
+}
+
 SrsLiveEntry::SrsLiveEntry(std::string m)
 {
     mount = m;

trunk/src/app/srs_app_http_stream.hpp

@@ -190,6 +190,13 @@ class SrsLiveStream : public ISrsHttpHandler
     virtual srs_error_t http_hooks_on_play(ISrsHttpMessage* r);
     virtual void http_hooks_on_stop(ISrsHttpMessage* r);
     virtual srs_error_t streaming_send_messages(ISrsBufferEncoder* enc, SrsSharedPtrMessage** msgs, int nb_msgs);
+    virtual srs_error_t filter_avc_nalu_sei(SrsRtmpFormat& format, SrsSharedPtrMessage** msgs, int& count);
+    virtual srs_error_t has_avc_nalu_sei(SrsRtmpFormat& format, SrsSharedPtrMessage** msgs, int& count, bool& has_sei);
+    virtual srs_error_t remux_avc_nalus(const SrsRtmpFormat& format,
+                                        const SrsSharedPtrMessage* msg,
+                                        SrsSharedPtrMessage** output_msg,
+                                        SrsSample* nalus,
+                                        size_t nalu_cout);
 };
 
 // The Live Entry, to handle HTTP Live Streaming.

trunk/src/kernel/srs_kernel_codec.hpp

@@ -1367,6 +1367,7 @@ class SrsFormat
     // The packet is muxed in FLV format, defined in flv specification.
     //          Demux the sps/pps from sequence header.
     //          Demux the samples from NALUs.
+    // TODO: rename to video_video_demux, because it can demux both h265 and h264.
     virtual srs_error_t video_avc_demux(SrsBuffer* stream, int64_t timestamp);
 #ifdef SRS_H265
 private:

trunk/src/kernel/srs_kernel_flv.cpp

@@ -258,6 +258,35 @@ srs_error_t SrsSharedPtrMessage::create(SrsMessageHeader* pheader, char* payload
     return err;
 }
 
+srs_error_t SrsSharedPtrMessage::create(const SrsSharedPtrMessage* msg, char* payload, int size)
+{
+        srs_error_t err = srs_success;
+
+    if (size < 0) {
+        return srs_error_new(ERROR_RTMP_MESSAGE_CREATE, "create message size=%d", size);
+    }
+
+    srs_assert(!ptr);
+    ptr = new SrsSharedPtrPayload();
+
+    // direct attach the data.
+    if (msg) {
+        ptr->header.message_type = msg->ptr->header.message_type;
+        ptr->header.payload_length = size;
+        ptr->header.perfer_cid = msg->ptr->header.perfer_cid;
+        this->timestamp = msg->timestamp;
+        this->stream_id = msg->stream_id;
+    }
+    ptr->payload = payload;
+    ptr->size = size;
+
+    // message can access it.
+    this->payload = ptr->payload;
+    this->size = ptr->size;
+
+    return err;
+}
+
 void SrsSharedPtrMessage::wrap(char* payload, int size)
 {
     srs_assert(!ptr);
@@ -298,18 +327,18 @@ bool SrsSharedPtrMessage::check(int stream_id)
     return false;
 }
 
-bool SrsSharedPtrMessage::is_av()
+bool SrsSharedPtrMessage::is_av() const
 {
     return ptr->header.message_type == RTMP_MSG_AudioMessage
     || ptr->header.message_type == RTMP_MSG_VideoMessage;
 }
 
-bool SrsSharedPtrMessage::is_audio()
+bool SrsSharedPtrMessage::is_audio() const
 {
     return ptr->header.message_type == RTMP_MSG_AudioMessage;
 }
 
-bool SrsSharedPtrMessage::is_video()
+bool SrsSharedPtrMessage::is_video() const
 {
     return ptr->header.message_type == RTMP_MSG_VideoMessage;
 }