Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Delta calculations #44

Closed
wants to merge 16 commits into from
Closed

Delta calculations #44

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
0a0b099
fix uLipSyncAnimator error
liudger May 2, 2023
7ba98d4
make sure it has hashed the animator
liudger May 3, 2023
ed85f3a
add minimum value tresshold
liudger May 10, 2023
f8570d0
add delta support in gui as optional
liudger Jun 5, 2023
8d3ba19
add support for 24 length array
liudger Jun 5, 2023
54c80f1
add buffer for storing previous melcepstrum
liudger Jun 5, 2023
7213b42
add logic for storing melcepstrum in buffer
liudger Jun 5, 2023
7d54d0d
add calculatedelta
liudger Jun 5, 2023
0181bce
add summary
liudger Jun 5, 2023
b5cb8a6
move jobHandle complete
liudger Jun 7, 2023
94d94b3
add simpler way to calculate buffer
liudger Jun 7, 2023
ab9176e
should return instead of break
liudger Jun 7, 2023
98ce12a
update threshold param
liudger Jun 7, 2023
602c56d
add attribute for aggressive inlining
liudger Jun 7, 2023
d4d52b5
add WriteOnly
liudger Jun 7, 2023
a83763b
use nativearray copy function
liudger Jun 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 19 additions & 4 deletions Assets/uLipSync/Editor/ProfileEditor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.Collections.Generic;
using System.Text;
using uLipSync.Debugging;
using System;

namespace uLipSync
{
Expand Down Expand Up @@ -36,11 +37,11 @@ public void Draw(bool showCalibration)
if (EditorUtil.SimpleFoldout("MFCC", true, "-uLipSync-Profile"))
{
EditorGUI.BeginChangeCheck();

++EditorGUI.indentLevel;
DrawMfccReorderableList(showCalibration);
--EditorGUI.indentLevel;

if (EditorGUI.EndChangeCheck())
{
EditorUtility.SetDirty(target);
Expand All @@ -62,6 +63,20 @@ public void Draw(bool showCalibration)
profile.UpdateMeansAndStandardization();
EditorUtility.SetDirty(target);
}
GUI.enabled = profile.mfccs.Count == 0;
if (profile.mfccs.Count != 0)
{
EditorGUILayout.HelpBox("Can't change delta setting when mfcc data exist." +
Environment.NewLine + "Create a new profile or delete the mfcc data", MessageType.Warning);
}
bool useDelta = EditorGUILayout.Toggle("Use Delta", profile.useDelta);
if (useDelta != profile.useDelta)
{
Undo.RecordObject(target, "Change Use Delta");
profile.useDelta = useDelta;
EditorUtility.SetDirty(target);
}
GUI.enabled = true;
EditorUtil.DrawProperty(serializedObject, nameof(profile.compareMethod));
profile.mfccDataCount = Mathf.Clamp(profile.mfccDataCount, 1, 256);
profile.melFilterBankChannels = Mathf.Clamp(profile.melFilterBankChannels, 12, 256);
Expand Down Expand Up @@ -118,7 +133,7 @@ void DrawMfccReorderableList(bool showCalibration)
if (_reorderableList == null)
{
_reorderableList = new ReorderableList(profile.mfccs, typeof(MfccData));
_reorderableList.drawHeaderCallback = rect =>
_reorderableList.drawHeaderCallback = rect =>
{
rect.xMin -= EditorGUI.indentLevel * 12f;
EditorGUI.LabelField(rect, "MFCCs");
Expand Down Expand Up @@ -182,7 +197,7 @@ void DrawMFCC(Rect position, int index, bool showCalibration)
if (!_texturePool.TryGetValue(data, out Texture2D tex)) tex = null;
tex = TextureCreator.CreateMfccTexture(tex, data, Common.MfccMinValue, Common.MfccMaxValue);
_texturePool[data] = tex;

var area = EditorGUI.IndentedRect(mfccPos);
area.height = data.mfccCalibrationDataList.Count * 3f;
GUI.DrawTexture(area, tex, ScaleMode.StretchToFill);
Expand Down
11 changes: 6 additions & 5 deletions Assets/uLipSync/Editor/uLipSyncAnimatorEditor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,13 @@ public override void OnInspectorGUI()
if (EditorUtil.Foldout("Animator Controller Parameters", true))
{
++EditorGUI.indentLevel;
if (anim.animator != null)
{
DrawAnimatorReorderableList();
if (anim.animator != null && anim.animator.isActiveAndEnabled)
{
DrawAnimatorReorderableList();
}
else
{
EditorGUILayout.HelpBox("Animator is not available.", MessageType.Warning);
{
EditorGUILayout.HelpBox("Animator is not available! To edit parameters open the prefab or have game object in scene.", MessageType.Warning);
}
--EditorGUI.indentLevel;
EditorGUILayout.Separator();
Expand Down Expand Up @@ -246,6 +246,7 @@ protected void DrawParameters()
EditorGUILayout.EndHorizontal();

EditorUtil.DrawProperty(serializedObject, nameof(anim.smoothness));
EditorUtil.DrawProperty(serializedObject, nameof(anim.minimalValueThreshold));
}
}

Expand Down
152 changes: 137 additions & 15 deletions Assets/uLipSync/Runtime/Core/Algorithm.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,24 @@
using Unity.Mathematics;
using Unity.Burst;
using Unity.Collections.LowLevel.Unsafe;
using System.Runtime.CompilerServices;

namespace uLipSync
{

[BurstCompile]
[BurstCompile(FloatMode = FloatMode.Fast, FloatPrecision = FloatPrecision.Low)]
public static unsafe class Algorithm
{
/// <summary>
/// Get the maximum value of the array.
/// </summary>
/// <param name="array">Array to get max from.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static float GetMaxValue(in NativeArray<float> array)
{
return GetMaxValue((float*)array.GetUnsafeReadOnlyPtr(), array.Length);
}

[BurstCompile]
static float GetMaxValue(float* array, int len)
{
Expand All @@ -25,6 +31,10 @@ static float GetMaxValue(float* array, int len)
return max;
}

/// <summary>
/// Get RMS volume.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static float GetRMSVolume(in NativeArray<float> array)
{
return GetRMSVolume((float*)array.GetUnsafeReadOnlyPtr(), array.Length);
Expand All @@ -41,13 +51,20 @@ static float GetRMSVolume(float *array, int len)
return math.sqrt(average / len);
}

/// <summary>
/// Copy ring buffer, startSrcIndex is the index of the oldest data.
/// </summary>
/// <param name="input">Source buffer.</param>
/// <param name="output">Destination buffer. This is a temporary buffer and needs to be disposed.</param>
/// <param name="startSrcIndex">Index of the oldest data.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void CopyRingBuffer(in NativeArray<float> input, out NativeArray<float> output, int startSrcIndex)
{
output = new NativeArray<float>(input.Length, Allocator.Temp);
CopyRingBuffer(
(float*)input.GetUnsafeReadOnlyPtr(),
(float*)output.GetUnsafePtr(),
input.Length,
(float*)input.GetUnsafeReadOnlyPtr(),
(float*)output.GetUnsafePtr(),
input.Length,
startSrcIndex);
}

Expand All @@ -60,6 +77,12 @@ static void CopyRingBuffer(float* input, float* output, int len, int startSrcInd
}
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
/// <summary>
/// Normalize array to the specified value.
/// </summary>
/// <param name="array">Array to normalize.</param>
/// <param name="value">Value to use for normalization.</param>
public static void Normalize(ref NativeArray<float> array, float value = 1f)
{
Normalize((float*)array.GetUnsafePtr(), array.Length, value);
Expand All @@ -77,6 +100,14 @@ static void Normalize(float* array, int len, float value = 1f)
}
}

/// <summary>
/// Low-pass filter, cutoff is normalized by sample rate.
/// </summary>
/// <param name="data">Data. </param>
/// <param name="sampleRate">Sample rate.</param>
/// <param name="cutoff">Cutoff frequency.</param>
/// <param name="range">Range of cutoff frequency.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void LowPassFilter(ref NativeArray<float> data, float sampleRate, float cutoff, float range)
{
cutoff = (cutoff - range) / sampleRate;
Expand Down Expand Up @@ -122,6 +153,14 @@ static void LowPassFilter(float* data, int len, float cutoff, float* tmp, float*
}
}

/// <summary>
/// Down sample the specified input.
/// </summary>
/// <param name="input">Input.</param>
/// <param name="output">Output. This is a temporary buffer and needs to be disposed.</param>
/// <param name="sampleRate">Sample rate.</param>
/// <param name="targetSampleRate">Target sample rate.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void DownSample(in NativeArray<float> input, out NativeArray<float> output, int sampleRate, int targetSampleRate)
{
if (sampleRate <= targetSampleRate)
Expand All @@ -133,8 +172,8 @@ public static void DownSample(in NativeArray<float> input, out NativeArray<float
int skip = sampleRate / targetSampleRate;
output = new NativeArray<float>(input.Length / skip, Allocator.Temp);
DownSample1(
(float*)input.GetUnsafeReadOnlyPtr(),
(float*)output.GetUnsafePtr(),
(float*)input.GetUnsafeReadOnlyPtr(),
(float*)output.GetUnsafePtr(),
output.Length,
skip);
}
Expand All @@ -144,9 +183,9 @@ public static void DownSample(in NativeArray<float> input, out NativeArray<float
int n = (int)math.round(input.Length / df);
output = new NativeArray<float>(n, Allocator.Temp);
DownSample2(
(float*)input.GetUnsafeReadOnlyPtr(),
(float*)input.GetUnsafeReadOnlyPtr(),
input.Length,
(float*)output.GetUnsafePtr(),
(float*)output.GetUnsafePtr(),
output.Length,
df);
}
Expand Down Expand Up @@ -176,6 +215,10 @@ static void DownSample2(float* input, int inputLen, float* output, int outputLen
}
}

/// <summary>
/// Pre-emphasis, which is a high-pass filter
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void PreEmphasis(ref NativeArray<float> data, float p)
{
var tmp = new NativeArray<float>(data, Allocator.Temp);
Expand All @@ -196,6 +239,7 @@ static void PreEmphasis(float* data, float* tmp, int len, float p)
}
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void HammingWindow(ref NativeArray<float> array)
{
HammingWindow((float*)array.GetUnsafePtr(), array.Length);
Expand All @@ -211,7 +255,12 @@ static void HammingWindow(float* array, int len)
}
}

public static void ZeroPadding(ref NativeArray<float> data, out NativeArray<float> dataWithPadding)
/// <summary>
/// Add zero padding to the begin and end of the data.
/// </summary>
/// <param name="data">Data.</param>
/// <param name="dataWithPadding">Data with padding. This is a temporary buffer and needs to be disposed.</param>
public static void ZeroPadding(ref NativeArray<float> data, out NativeArray<float> dataWithPadding)
{
int N = data.Length;
dataWithPadding = new NativeArray<float>(N * 2, Allocator.Temp);
Expand All @@ -226,6 +275,12 @@ public static void ZeroPadding(ref NativeArray<float> data, out NativeArray<floa
UnsafeUtility.MemSet((float*)slice3.GetUnsafePtr<float>(), 0, sizeof(float) * slice1.Length);
}

/// <summary>
/// Fast Fourier transform.
/// </summary>
/// <param name="data">Data.</param>
/// <param name="spectrum">Spectrum. This is a temporary buffer and needs to be disposed.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void FFT(in NativeArray<float> data, out NativeArray<float> spectrum)
{
int N = data.Length;
Expand Down Expand Up @@ -298,8 +353,14 @@ static void _FFT(float* spectrumRe, float* spectrumIm, int N)
oddIm.Dispose();
}

/// <summary>
/// Convert frequency to mel frequency by subdividing the mel scale into melDiv parts.
/// </summary>
/// <param name="spectrum">Spectrum.</param>
/// <param name="melSpectrum">Mel spectrum. This is a temporary buffer and needs to be disposed.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void MelFilterBank(
in NativeArray<float> spectrum,
in NativeArray<float> spectrum,
out NativeArray<float> melSpectrum,
float sampleRate,
int melDiv)
Expand All @@ -315,7 +376,7 @@ public static void MelFilterBank(

[BurstCompile]
static void MelFilterBank(
float* spectrum,
float* spectrum,
float* melSpectrum,
int len,
float sampleRate,
Expand Down Expand Up @@ -345,8 +406,8 @@ static void MelFilterBank(
for (int i = iBegin + 1; i <= iEnd; ++i)
{
float f = df * i;
float a = (i < iCenter) ?
(f - fBegin) / (fCenter - fBegin) :
float a = (i < iCenter) ?
(f - fBegin) / (fCenter - fBegin) :
(fEnd - f) / (fEnd - fCenter);
a /= (fEnd - fBegin) * 0.5f;
sum += a * spectrum[i];
Expand All @@ -355,6 +416,10 @@ static void MelFilterBank(
}
}

/// <summary>
/// Convert power spectrum to decibel.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void PowerToDb(ref NativeArray<float> array)
{
PowerToDb((float*)array.GetUnsafePtr(), array.Length);
Expand Down Expand Up @@ -383,13 +448,19 @@ static float ToHz(float mel, bool slaney = false)
return 700f * (math.exp(mel / a) - 1f);
}

/// <summary>
/// Discrete Cosine Transform.
/// </summary>
/// <param name="spectrum">Spectrum.</param>
/// <param name="cepstrum">Cepstrum. This is a temporary buffer and needs to be disposed.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void DCT(
in NativeArray<float> spectrum,
out NativeArray<float> cepstrum)
{
cepstrum = new NativeArray<float>(spectrum.Length, Allocator.Temp);
DCT(
(float*)spectrum.GetUnsafeReadOnlyPtr(),
(float*)spectrum.GetUnsafeReadOnlyPtr(),
(float*)cepstrum.GetUnsafePtr(),
spectrum.Length);
}
Expand All @@ -413,11 +484,62 @@ static void DCT(
}
}

/// <summary>
/// Calculate delta coefficients.
/// </summary>
/// <param name="buffer">MFCC buffer.</param>
/// <param name="delta">Delta coefficients. This is a temporary buffer and needs to be disposed.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void CalculateDelta(
in NativeArray<float> buffer,
out NativeArray<float> delta,
int bufferSize = 3,
int numCoefficients = 12)
{
delta = new NativeArray<float>(numCoefficients, Allocator.Temp);

CalculateDelta(
(float*)buffer.GetUnsafeReadOnlyPtr(),
(float*)delta.GetUnsafePtr(),
bufferSize,
numCoefficients);
}

[BurstCompile]
private static unsafe void CalculateDelta(
float* buffer,
float* delta,
int bufferSize,
int numCoefficients)
{
for (int i = 0; i < numCoefficients; i++)
{
// Calculate delta
float numerator = 0f;
float denominator = 0f;
int index = 0;

for (int j = 0; j < bufferSize-1; j++)
{
float frameMfcc = buffer[j * numCoefficients + i];

float weight = index * index;
numerator += weight * frameMfcc;
denominator += weight;
index++;
}

delta[i] = numerator / (2 * denominator);
}
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static float Norm(in NativeArray<float> array)
{
return Norm((float*)array.GetUnsafeReadOnlyPtr(), array.Length);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static float Norm(in NativeSlice<float> slice)
{
return Norm((float*)slice.GetUnsafeReadOnlyPtr(), slice.Length);
Expand Down
Loading