MT#55283 remove unsupported codecs w/ audio player

Remove all codecs that cannot be transcoded to when the audio player is in
use.

Add safety to make_transcoder to return failure in case an unsupported
codec is requested for transcoding.

Convert leftover passthrough handlers to SSRC passthrough if there are
any (shouldn't be).

closes #1858

Change-Id: I1822e48723622d550624c7355a1acfbf8ca38eb8
(cherry picked from commit 4573f32)

daemon/codec.c

@@ -437,14 +437,15 @@ static void __reset_sequencer(void *p, void *dummy) {
 		g_hash_table_destroy(s->sequencers);
 	s->sequencers = NULL;
 }
-static void __make_transcoder_full(struct codec_handler *handler, rtp_payload_type *dest,
+static bool __make_transcoder_full(struct codec_handler *handler, rtp_payload_type *dest,
 		GHashTable *output_transcoders, int dtmf_payload_type, bool pcm_dtmf_detect,
 		int cn_payload_type, int (*packet_decoded)(decoder_t *, AVFrame *, void *, void *),
 		struct ssrc_entry *(*ssrc_handler_new_func)(void *p))
 {
-	assert(handler->source_pt.codec_def != NULL);
-
-	assert(dest->codec_def != NULL);
+	if (!handler->source_pt.codec_def)
+		return false;
+	if (!dest->codec_def)
+		return false;
 
 	// don't reset handler if it already matches what we want
 	if (!handler->transcoder)
@@ -542,6 +543,8 @@ static void __make_transcoder_full(struct codec_handler *handler, rtp_payload_ty
 			g_hash_table_insert(output_transcoders, GINT_TO_POINTER(dest->payload_type), handler);
 		handler->output_handler = handler; // make sure we don't have a stale pointer
 	}
+
+	return true;
 }
 static void __make_transcoder(struct codec_handler *handler, rtp_payload_type *dest,
 		GHashTable *output_transcoders, int dtmf_payload_type, bool pcm_dtmf_detect,
@@ -550,10 +553,10 @@ static void __make_transcoder(struct codec_handler *handler, rtp_payload_type *d
 	__make_transcoder_full(handler, dest, output_transcoders, dtmf_payload_type, pcm_dtmf_detect,
 			cn_payload_type, packet_decoded_fifo, __ssrc_handler_transcode_new);
 }
-static void __make_audio_player_decoder(struct codec_handler *handler, rtp_payload_type *dest,
+static bool __make_audio_player_decoder(struct codec_handler *handler, rtp_payload_type *dest,
 		bool pcm_dtmf_detect)
 {
-	__make_transcoder_full(handler, dest, NULL, -1, pcm_dtmf_detect, -1, packet_decoded_audio_player,
+	return __make_transcoder_full(handler, dest, NULL, -1, pcm_dtmf_detect, -1, packet_decoded_audio_player,
 			__ssrc_handler_decode_new);
 }
 
@@ -1477,26 +1480,25 @@ sink_pt_fixed:;
 		if (a.reset_transcoding && ms)
 			ms->attrs.transcoding = 1;
 
-		if (!use_audio_player) {
-			// we have to translate RTCP packets
-			receiver->rtcp_handler = rtcp_transcode_handler;
-
-			for (__auto_type l = receiver->codecs.codec_prefs.head; l; ) {
-				rtp_payload_type *pt = l->data;
-
-				if (pt->codec_def) {
-					// supported
-					l = l->next;
-					continue;
-				}
+		for (__auto_type l = receiver->codecs.codec_prefs.head; l; ) {
+			rtp_payload_type *pt = l->data;
 
-				ilogs(codec, LOG_DEBUG, "Stripping unsupported codec " STR_FORMAT
-						" due to active transcoding",
-						STR_FMT(&pt->encoding));
-				codec_touched(&receiver->codecs, pt);
-				l = __codec_store_delete_link(l, &receiver->codecs);
+			if (pt->codec_def) {
+				// supported
+				l = l->next;
+				continue;
 			}
 
+			ilogs(codec, LOG_DEBUG, "Stripping unsupported codec " STR_FORMAT
+					" due to active transcoding",
+					STR_FMT(&pt->encoding));
+			codec_touched(&receiver->codecs, pt);
+			l = __codec_store_delete_link(l, &receiver->codecs);
+		}
+
+		if (!use_audio_player) {
+			// we have to translate RTCP packets
+			receiver->rtcp_handler = rtcp_transcode_handler;
 
 			// at least some payload types will be transcoded, which will result in SSRC
 			// change. for payload types which we don't actually transcode, we still
@@ -1506,7 +1508,6 @@ sink_pt_fixed:;
 				__convert_passthrough_ssrc(handler);
 				passthrough_handlers = g_slist_delete_link(passthrough_handlers,
 						passthrough_handlers);
-
 			}
 		}
 		else {
@@ -1516,7 +1517,8 @@ sink_pt_fixed:;
 			// change all passthrough handlers also to transcoders
 			while (passthrough_handlers) {
 				struct codec_handler *handler = passthrough_handlers->data;
-				__make_audio_player_decoder(handler, pref_dest_codec, false);
+				if (!__make_audio_player_decoder(handler, pref_dest_codec, false))
+					__convert_passthrough_ssrc(handler);
 				passthrough_handlers = g_slist_delete_link(passthrough_handlers,
 						passthrough_handlers);