From 5a795d3b305ae02a28ff845175bf5d2aef9bd194 Mon Sep 17 00:00:00 2001 From: ShaneRH Date: Wed, 10 Feb 2021 09:27:45 +0200 Subject: [PATCH] Performance improvements changes by @RenderHeadsSte General optimisations (reducing lock size, improving circular buffer performance) Added read-ahead buffer for audio do avoid empty buffer. --- .../Runtime/Component/NdiReceiver.cs | 130 ++++++++++------ .../CircularBuffer/CircularBuffer.cs | 145 +++++++++++++++--- 2 files changed, 204 insertions(+), 71 deletions(-) diff --git a/Packages/jp.keijiro.klak.ndi/Runtime/Component/NdiReceiver.cs b/Packages/jp.keijiro.klak.ndi/Runtime/Component/NdiReceiver.cs index 6cbf0c4f..bba43c74 100644 --- a/Packages/jp.keijiro.klak.ndi/Runtime/Component/NdiReceiver.cs +++ b/Packages/jp.keijiro.klak.ndi/Runtime/Component/NdiReceiver.cs @@ -212,9 +212,17 @@ void ProcessStatusChange(System.Object data) #region Audio implementation - private readonly object audioBufferLock = new object(); - private CircularBuffer audioBuffer; - private const int BUFFER_SIZE = 4096; + private readonly object audioBufferLock = new object(); + private const int BUFFER_SIZE = 1024 * 32; + private CircularBuffer audioBuffer = new CircularBuffer(BUFFER_SIZE); + // + private bool m_bWaitForBufferFill = true; + private const int m_iMinBufferAheadFrames = 4; + // + private NativeArray m_aTempAudioPullBuffer; + private Interop.AudioFrameInterleaved interleavedAudio = new Interop.AudioFrameInterleaved(); + // + private float[] m_aTempSamplesArray = new float[ 1024 * 32 ]; void PrepareAudioSource(Interop.AudioFrame audioFrame) { @@ -232,14 +240,6 @@ void PrepareAudioSource(Interop.AudioFrame audioFrame) // Create a AudioClip that matches the incomming frame audioClip = AudioClip.Create("NdiReceiver Audio", audioFrame.SampleRate, audioFrame.NoChannels, audioFrame.SampleRate, true); - - lock (audioBufferLock) - { - if (audioBuffer == null || audioBuffer.Capacity != audioFrame.SampleRate) - { - audioBuffer = new CircularBuffer(BUFFER_SIZE * audioFrame.NoChannels); - } - } } audioSource.loop = true; @@ -249,23 +249,43 @@ void PrepareAudioSource(Interop.AudioFrame audioFrame) void OnAudioFilterRead(float[] data, int channels) { + int length = data.Length; + + // STE: Waiting for enough read ahead buffer frames? + if (m_bWaitForBufferFill) + { + // Are we good yet? + // Should we be protecting audioBuffer.Size here? + m_bWaitForBufferFill = ( audioBuffer.Size < (length * m_iMinBufferAheadFrames) ); + + // Early out if not enough in the buffer still + if (m_bWaitForBufferFill) + { + return; + } + } + + bool bPreviousWaitForBufferFill = m_bWaitForBufferFill; + int iAudioBufferSize = 0; + + // STE: Lock buffer for the smallest amount of time lock (audioBufferLock) { - int length = data.Length; + iAudioBufferSize = audioBuffer.Size; - for (int i = 0; i < length; i++) + // If we do not have enough data for a single frame then we will want to buffer up some read-ahead audio data. This will cause a longer gap in the audio playback, but this is better than more intermittent glitches I think + m_bWaitForBufferFill = (iAudioBufferSize < length); + if( !m_bWaitForBufferFill ) { - if (audioBuffer.IsEmpty) - { - data[i] = 0.0f; - } - else - { - data[i] = audioBuffer.Front(); - audioBuffer.PopFront(); - } + audioBuffer.Front( ref data, data.Length ); + audioBuffer.PopFront( data.Length ); } } + + if ( m_bWaitForBufferFill && !bPreviousWaitForBufferFill ) + { + Debug.Log("NOT ENOUGH AUDIO : OnAudioFilterRead: data.Length = " + data.Length + "| audioBuffer.Size = " + iAudioBufferSize); + } } void FillAudioBuffer(Interop.AudioFrame audio) @@ -275,43 +295,53 @@ void FillAudioBuffer(Interop.AudioFrame audio) return; } - lock (audioBufferLock) + // Converted from NDI C# Managed sample code + // we're working in bytes, so take the size of a 32 bit sample (float) into account + int sizeInBytes = audio.NoSamples * audio.NoChannels * sizeof(float); + + // Unity is expecting interleaved audio and NDI uses planar. + // create an interleaved frame and convert from the one we received + interleavedAudio.SampleRate = audio.SampleRate; + interleavedAudio.NoChannels = audio.NoChannels; + interleavedAudio.NoSamples = audio.NoSamples; + interleavedAudio.Timecode = audio.Timecode; + + // allocate native array to copy interleaved data into + unsafe { - if (audioBuffer == null) + if( m_aTempAudioPullBuffer == null || m_aTempAudioPullBuffer.Length < sizeInBytes) { - audioBuffer = new CircularBuffer(BUFFER_SIZE * audio.NoChannels); + m_aTempAudioPullBuffer = new NativeArray(sizeInBytes, Allocator.Persistent, NativeArrayOptions.UninitializedMemory); } - // Converted from NDI C# Managed sample code - // we're working in bytes, so take the size of a 32 bit sample (float) into account - int sizeInBytes = audio.NoSamples * audio.NoChannels * sizeof(float); - - // Unity is expecting interleaved audio and NDI uses planar. - // create an interleaved frame and convert from the one we received - Interop.AudioFrameInterleaved interleavedAudio = new Interop.AudioFrameInterleaved() + interleavedAudio.Data = (IntPtr)m_aTempAudioPullBuffer.GetUnsafePtr(); + if ( interleavedAudio.Data != null ) { - SampleRate = audio.SampleRate, - NoChannels = audio.NoChannels, - NoSamples = audio.NoSamples, - Timecode = audio.Timecode - }; - - // allocate native array to copy interleaved data into - unsafe - { - using (var nativeArray = new NativeArray(sizeInBytes, Allocator.TempJob, NativeArrayOptions.UninitializedMemory)) - { - interleavedAudio.Data = (IntPtr)nativeArray.GetUnsafePtr(); + // Convert from float planar to float interleaved audio + _recv.AudioFrameToInterleaved(ref audio, ref interleavedAudio); - // Convert from float planar to float interleaved audio - _recv.AudioFrameToInterleaved(ref audio, ref interleavedAudio); + var totalSamples = interleavedAudio.NoSamples * interleavedAudio.NoChannels; + void* audioDataPtr = interleavedAudio.Data.ToPointer(); - var totalSamples = interleavedAudio.NoSamples * interleavedAudio.NoChannels; - void* audioDataPtr = interleavedAudio.Data.ToPointer(); + if( audioDataPtr != null ) + { + // Grab data from native array + if( m_aTempSamplesArray == null || m_aTempSamplesArray.Length < totalSamples ) + { + m_aTempSamplesArray = new float[ totalSamples ]; + } + if( m_aTempSamplesArray != null ) + { + for (int i = 0; i < totalSamples; i++) + { + m_aTempSamplesArray[ i ] = UnsafeUtility.ReadArrayElement( audioDataPtr, i ); + } + } - for (int i = 0; i < totalSamples; i++) + // Copy new sample data into the circular array + lock (audioBufferLock) { - audioBuffer.PushBack(UnsafeUtility.ReadArrayElement(audioDataPtr, i)); + audioBuffer.PushBack( m_aTempSamplesArray, totalSamples ); } } } diff --git a/Packages/jp.keijiro.klak.ndi/Runtime/ThirdParty/CircularBuffer/CircularBuffer.cs b/Packages/jp.keijiro.klak.ndi/Runtime/ThirdParty/CircularBuffer/CircularBuffer.cs index 0434eef2..d430bcd8 100644 --- a/Packages/jp.keijiro.klak.ndi/Runtime/ThirdParty/CircularBuffer/CircularBuffer.cs +++ b/Packages/jp.keijiro.klak.ndi/Runtime/ThirdParty/CircularBuffer/CircularBuffer.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; using System.Collections; +using UnityEngine; + namespace CircularBuffer { @@ -115,11 +117,42 @@ public T Front() return _buffer[_start]; } - /// - /// Element at the back of the buffer - this[Size - 1]. - /// - /// The value of the element of type T at the back of the buffer. - public T Back() + /// + /// Element at the front of the buffer - this[0]. + /// + /// Copies the elements from the front of the buffer to a supplied array. + public void Front( ref T[] aToArray, int iRequested) + { + ThrowIfEmpty(); + + // Can pull all the elements? + if(_size < iRequested ) + { + throw new InvalidOperationException("Not enough elements in the buffer"); + } + + int iToEndOfBuffer = _buffer.Length - _start; + if ( iRequested <= iToEndOfBuffer) + { + // Copy out in one shot + Array.Copy( _buffer, _start, aToArray, 0, iRequested ); + } + else + { + // Copy from current head to end of buffer + Array.Copy(_buffer, _start, aToArray, 0, iToEndOfBuffer); + + // Copy from start of buffer + int iRemaining = iRequested - iToEndOfBuffer; + Array.Copy(_buffer, 0, aToArray, iToEndOfBuffer, iRemaining); + } + } + + /// + /// Element at the back of the buffer - this[Size - 1]. + /// + /// The value of the element of type T at the back of the buffer. + public T Back() { ThrowIfEmpty(); return _buffer[(_end != 0 ? _end : Capacity) - 1]; @@ -179,15 +212,64 @@ public void PushBack(T item) } } - /// - /// Pushes a new element to the front of the buffer. Front()/this[0] - /// will now return this element. - /// - /// When the buffer is full, the element at Back()/this[Size-1] will be - /// popped to allow for this new element to fit. - /// - /// Item to push to the front of the buffer - public void PushFront(T item) + /// + /// Pushes a new element to the back of the buffer. Back()/this[Size-1] + /// will now return this element. + /// + /// When the buffer is full, the element at Front()/this[0] will be + /// popped to allow for this new element to fit. + /// + /// Item to push to the back of the buffer + public void PushBack(T[] aitems, int iToAdd) + { + // Cannot copy more than a full buffers worth + if( iToAdd > _buffer.Length ) + { + throw new InvalidOperationException("Cannot copy more than a full buffers worth"); + } + + // Pushing more than we have room for? + bool bOverrun = ( iToAdd > (_buffer.Length - _size) ); + + // Copy in a single chunk? + int iToEndOfBuffer = _buffer.Length - _end; + if (iToAdd <= iToEndOfBuffer) + { + // Copy out in one shot + Array.Copy(aitems, 0, _buffer, _end, iToAdd); + } + else + { + // Copy to the end of the buffer + Array.Copy(aitems, 0, _buffer, _end, iToEndOfBuffer); + + // Copy to start of buffer + int iRemaining = iToAdd - iToEndOfBuffer; + Array.Copy(aitems, iToEndOfBuffer, _buffer, 0, iRemaining); + } + + _end = (_end + iToAdd) % _buffer.Length; + if ( bOverrun ) + { + _start = _end; + } + + _size += iToAdd; + if (_size > _buffer.Length) + { + _size = _buffer.Length; + } + } + + /// + /// Pushes a new element to the front of the buffer. Front()/this[0] + /// will now return this element. + /// + /// When the buffer is full, the element at Back()/this[Size-1] will be + /// popped to allow for this new element to fit. + /// + /// Item to push to the front of the buffer + public void PushFront(T item) { if (IsFull) { @@ -227,13 +309,34 @@ public void PopFront() --_size; } - /// - /// Copies the buffer contents to an array, according to the logical - /// contents of the buffer (i.e. independent of the internal - /// order/contents) - /// - /// A new array with a copy of the buffer contents. - public T[] ToArray() + /// + /// Removes elements at the front of the buffer. Decreasing the + /// Buffer size by iRequested. + /// + public void PopFront(int iRequested) + { + ThrowIfEmpty("Cannot take elements from an empty buffer."); + + // Enough elements + if (_size < iRequested) + { + throw new InvalidOperationException("Not enough elements in the buffer to pop"); + } + + // TODO: Clear elements? Really don't need to +// _buffer[_start] = default(T); + + _start = ( _start + iRequested ) % _buffer.Length; + _size -= iRequested; + } + + /// + /// Copies the buffer contents to an array, according to the logical + /// contents of the buffer (i.e. independent of the internal + /// order/contents) + /// + /// A new array with a copy of the buffer contents. + public T[] ToArray() { T[] newArray = new T[Size]; int newArrayOffset = 0;