Skip to content

Audio performance improvements #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ShaneRH wants to merge 2 commits into
base: audio-receiver
Choose a base branch
from
Open

Audio performance improvements #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3b191ea
Merge pull request #1 from darktable/audio-receiver
ShaneRH Feb 10, 2021
5a795d3
Performance improvements
ShaneRH Feb 10, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 80 additions & 50 deletions Packages/jp.keijiro.klak.ndi/Runtime/Component/NdiReceiver.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,9 +212,17 @@ void ProcessStatusChange(System.Object data)

#region Audio implementation

private readonly object audioBufferLock = new object();
private CircularBuffer<float> audioBuffer;
private const int BUFFER_SIZE = 4096;
private readonly object audioBufferLock = new object();
private const int BUFFER_SIZE = 1024 * 32;
private CircularBuffer<float> audioBuffer = new CircularBuffer<float>(BUFFER_SIZE);
//
private bool m_bWaitForBufferFill = true;
private const int m_iMinBufferAheadFrames = 4;
//
private NativeArray<byte> m_aTempAudioPullBuffer;
private Interop.AudioFrameInterleaved interleavedAudio = new Interop.AudioFrameInterleaved();
//
private float[] m_aTempSamplesArray = new float[ 1024 * 32 ];

void PrepareAudioSource(Interop.AudioFrame audioFrame)
{
Expand All @@ -232,14 +240,6 @@ void PrepareAudioSource(Interop.AudioFrame audioFrame)

// Create a AudioClip that matches the incomming frame
audioClip = AudioClip.Create("NdiReceiver Audio", audioFrame.SampleRate, audioFrame.NoChannels, audioFrame.SampleRate, true);

lock (audioBufferLock)
{
if (audioBuffer == null || audioBuffer.Capacity != audioFrame.SampleRate)
{
audioBuffer = new CircularBuffer<float>(BUFFER_SIZE * audioFrame.NoChannels);
}
}
}

audioSource.loop = true;
Expand All @@ -249,23 +249,43 @@ void PrepareAudioSource(Interop.AudioFrame audioFrame)

void OnAudioFilterRead(float[] data, int channels)
{
int length = data.Length;

// STE: Waiting for enough read ahead buffer frames?
if (m_bWaitForBufferFill)
{
// Are we good yet?
// Should we be protecting audioBuffer.Size here?
m_bWaitForBufferFill = ( audioBuffer.Size < (length * m_iMinBufferAheadFrames) );

// Early out if not enough in the buffer still
if (m_bWaitForBufferFill)
{
return;
}
}

bool bPreviousWaitForBufferFill = m_bWaitForBufferFill;
int iAudioBufferSize = 0;

// STE: Lock buffer for the smallest amount of time
lock (audioBufferLock)
{
int length = data.Length;
iAudioBufferSize = audioBuffer.Size;

for (int i = 0; i < length; i++)
// If we do not have enough data for a single frame then we will want to buffer up some read-ahead audio data. This will cause a longer gap in the audio playback, but this is better than more intermittent glitches I think
m_bWaitForBufferFill = (iAudioBufferSize < length);
if( !m_bWaitForBufferFill )
{
if (audioBuffer.IsEmpty)
{
data[i] = 0.0f;
}
else
{
data[i] = audioBuffer.Front();
audioBuffer.PopFront();
}
audioBuffer.Front( ref data, data.Length );
audioBuffer.PopFront( data.Length );
}
}

if ( m_bWaitForBufferFill && !bPreviousWaitForBufferFill )
{
Debug.Log("NOT ENOUGH AUDIO : OnAudioFilterRead: data.Length = " + data.Length + "| audioBuffer.Size = " + iAudioBufferSize);
}
}

void FillAudioBuffer(Interop.AudioFrame audio)
Expand All @@ -275,43 +295,53 @@ void FillAudioBuffer(Interop.AudioFrame audio)
return;
}

lock (audioBufferLock)
// Converted from NDI C# Managed sample code
// we're working in bytes, so take the size of a 32 bit sample (float) into account
int sizeInBytes = audio.NoSamples * audio.NoChannels * sizeof(float);

// Unity is expecting interleaved audio and NDI uses planar.
// create an interleaved frame and convert from the one we received
interleavedAudio.SampleRate = audio.SampleRate;
interleavedAudio.NoChannels = audio.NoChannels;
interleavedAudio.NoSamples = audio.NoSamples;
interleavedAudio.Timecode = audio.Timecode;

// allocate native array to copy interleaved data into
unsafe
{
if (audioBuffer == null)
if( m_aTempAudioPullBuffer == null || m_aTempAudioPullBuffer.Length < sizeInBytes)
{
audioBuffer = new CircularBuffer<float>(BUFFER_SIZE * audio.NoChannels);
m_aTempAudioPullBuffer = new NativeArray<byte>(sizeInBytes, Allocator.Persistent, NativeArrayOptions.UninitializedMemory);
}

// Converted from NDI C# Managed sample code
// we're working in bytes, so take the size of a 32 bit sample (float) into account
int sizeInBytes = audio.NoSamples * audio.NoChannels * sizeof(float);

// Unity is expecting interleaved audio and NDI uses planar.
// create an interleaved frame and convert from the one we received
Interop.AudioFrameInterleaved interleavedAudio = new Interop.AudioFrameInterleaved()
interleavedAudio.Data = (IntPtr)m_aTempAudioPullBuffer.GetUnsafePtr();
if ( interleavedAudio.Data != null )
{
SampleRate = audio.SampleRate,
NoChannels = audio.NoChannels,
NoSamples = audio.NoSamples,
Timecode = audio.Timecode
};

// allocate native array to copy interleaved data into
unsafe
{
using (var nativeArray = new NativeArray<byte>(sizeInBytes, Allocator.TempJob, NativeArrayOptions.UninitializedMemory))
{
interleavedAudio.Data = (IntPtr)nativeArray.GetUnsafePtr();
// Convert from float planar to float interleaved audio
_recv.AudioFrameToInterleaved(ref audio, ref interleavedAudio);

// Convert from float planar to float interleaved audio
_recv.AudioFrameToInterleaved(ref audio, ref interleavedAudio);
var totalSamples = interleavedAudio.NoSamples * interleavedAudio.NoChannels;
void* audioDataPtr = interleavedAudio.Data.ToPointer();

var totalSamples = interleavedAudio.NoSamples * interleavedAudio.NoChannels;
void* audioDataPtr = interleavedAudio.Data.ToPointer();
if( audioDataPtr != null )
{
// Grab data from native array
if( m_aTempSamplesArray == null || m_aTempSamplesArray.Length < totalSamples )
{
m_aTempSamplesArray = new float[ totalSamples ];
}
if( m_aTempSamplesArray != null )
{
for (int i = 0; i < totalSamples; i++)
{
m_aTempSamplesArray[ i ] = UnsafeUtility.ReadArrayElement<float>( audioDataPtr, i );
}
}

for (int i = 0; i < totalSamples; i++)
// Copy new sample data into the circular array
lock (audioBufferLock)
{
audioBuffer.PushBack(UnsafeUtility.ReadArrayElement<float>(audioDataPtr, i));
audioBuffer.PushBack( m_aTempSamplesArray, totalSamples );
}
}
}
Expand Down
145 changes: 124 additions & 21 deletions Packages/jp.keijiro.klak.ndi/Runtime/ThirdParty/CircularBuffer/CircularBuffer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
using System;
using System.Collections.Generic;
using System.Collections;
using UnityEngine;


namespace CircularBuffer
{
Expand Down Expand Up @@ -115,11 +117,42 @@ public T Front()
return _buffer[_start];
}

/// <summary>
/// Element at the back of the buffer - this[Size - 1].
/// </summary>
/// <returns>The value of the element of type T at the back of the buffer.</returns>
public T Back()
/// <summary>
/// Element at the front of the buffer - this[0].
/// </summary>
/// <returns>Copies the elements from the front of the buffer to a supplied array.</returns>
public void Front( ref T[] aToArray, int iRequested)
{
ThrowIfEmpty();

// Can pull all the elements?
if(_size < iRequested )
{
throw new InvalidOperationException("Not enough elements in the buffer");
}

int iToEndOfBuffer = _buffer.Length - _start;
if ( iRequested <= iToEndOfBuffer)
{
// Copy out in one shot
Array.Copy( _buffer, _start, aToArray, 0, iRequested );
}
else
{
// Copy from current head to end of buffer
Array.Copy(_buffer, _start, aToArray, 0, iToEndOfBuffer);

// Copy from start of buffer
int iRemaining = iRequested - iToEndOfBuffer;
Array.Copy(_buffer, 0, aToArray, iToEndOfBuffer, iRemaining);
}
}

/// <summary>
/// Element at the back of the buffer - this[Size - 1].
/// </summary>
/// <returns>The value of the element of type T at the back of the buffer.</returns>
public T Back()
{
ThrowIfEmpty();
return _buffer[(_end != 0 ? _end : Capacity) - 1];
Expand Down Expand Up @@ -179,15 +212,64 @@ public void PushBack(T item)
}
}

/// <summary>
/// Pushes a new element to the front of the buffer. Front()/this[0]
/// will now return this element.
///
/// When the buffer is full, the element at Back()/this[Size-1] will be
/// popped to allow for this new element to fit.
/// </summary>
/// <param name="item">Item to push to the front of the buffer</param>
public void PushFront(T item)
/// <summary>
/// Pushes a new element to the back of the buffer. Back()/this[Size-1]
/// will now return this element.
///
/// When the buffer is full, the element at Front()/this[0] will be
/// popped to allow for this new element to fit.
/// </summary>
/// <param name="item">Item to push to the back of the buffer</param>
public void PushBack(T[] aitems, int iToAdd)
{
// Cannot copy more than a full buffers worth
if( iToAdd > _buffer.Length )
{
throw new InvalidOperationException("Cannot copy more than a full buffers worth");
}

// Pushing more than we have room for?
bool bOverrun = ( iToAdd > (_buffer.Length - _size) );

// Copy in a single chunk?
int iToEndOfBuffer = _buffer.Length - _end;
if (iToAdd <= iToEndOfBuffer)
{
// Copy out in one shot
Array.Copy(aitems, 0, _buffer, _end, iToAdd);
}
else
{
// Copy to the end of the buffer
Array.Copy(aitems, 0, _buffer, _end, iToEndOfBuffer);

// Copy to start of buffer
int iRemaining = iToAdd - iToEndOfBuffer;
Array.Copy(aitems, iToEndOfBuffer, _buffer, 0, iRemaining);
}

_end = (_end + iToAdd) % _buffer.Length;
if ( bOverrun )
{
_start = _end;
}

_size += iToAdd;
if (_size > _buffer.Length)
{
_size = _buffer.Length;
}
}

/// <summary>
/// Pushes a new element to the front of the buffer. Front()/this[0]
/// will now return this element.
///
/// When the buffer is full, the element at Back()/this[Size-1] will be
/// popped to allow for this new element to fit.
/// </summary>
/// <param name="item">Item to push to the front of the buffer</param>
public void PushFront(T item)
{
if (IsFull)
{
Expand Down Expand Up @@ -227,13 +309,34 @@ public void PopFront()
--_size;
}

/// <summary>
/// Copies the buffer contents to an array, according to the logical
/// contents of the buffer (i.e. independent of the internal
/// order/contents)
/// </summary>
/// <returns>A new array with a copy of the buffer contents.</returns>
public T[] ToArray()
/// <summary>
/// Removes elements at the front of the buffer. Decreasing the
/// Buffer size by iRequested.
/// </summary>
public void PopFront(int iRequested)
{
ThrowIfEmpty("Cannot take elements from an empty buffer.");

// Enough elements
if (_size < iRequested)
{
throw new InvalidOperationException("Not enough elements in the buffer to pop");
}

// TODO: Clear elements? Really don't need to
// _buffer[_start] = default(T);

_start = ( _start + iRequested ) % _buffer.Length;
_size -= iRequested;
}

/// <summary>
/// Copies the buffer contents to an array, according to the logical
/// contents of the buffer (i.e. independent of the internal
/// order/contents)
/// </summary>
/// <returns>A new array with a copy of the buffer contents.</returns>
public T[] ToArray()
{
T[] newArray = new T[Size];
int newArrayOffset = 0;
Expand Down