From f5bc387d0f9e41a30fa2915bc41c4af8e13845b9 Mon Sep 17 00:00:00 2001 From: cadlaxa Date: Sat, 16 May 2026 23:28:16 +0800 Subject: [PATCH 01/18] morphing curves --- OpenUtau.Core/Classic/ClassicRenderer.cs | 1111 ++++++++++++++++- OpenUtau.Core/Classic/SpectralMorpher.cs | 124 ++ OpenUtau.Core/OpenUtau.Core.csproj | 1 + OpenUtau.Core/Render/RenderPhrase.cs | 2 +- OpenUtau.Core/Ustx/UExpression.cs | 1 + OpenUtau.Core/Util/Preferences.cs | 2 + OpenUtau/Controls/ExpressionCanvas.cs | 2 +- OpenUtau/Controls/PianoRoll.axaml.cs | 4 +- OpenUtau/Strings/Strings.axaml | 3 + OpenUtau/ViewModels/ExpressionsViewModel.cs | 10 +- .../ViewModels/NotePropertiesViewModel.cs | 2 +- OpenUtau/ViewModels/NotesViewModel.cs | 2 +- OpenUtau/ViewModels/PasteParamViewModel.cs | 2 +- OpenUtau/ViewModels/PreferencesViewModel.cs | 15 +- OpenUtau/Views/ExpressionsDialog.axaml | 1 + OpenUtau/Views/NoteEditStates.cs | 4 +- OpenUtau/Views/PreferencesDialog.axaml | 8 + 17 files changed, 1219 insertions(+), 75 deletions(-) create mode 100644 OpenUtau.Core/Classic/SpectralMorpher.cs diff --git a/OpenUtau.Core/Classic/ClassicRenderer.cs b/OpenUtau.Core/Classic/ClassicRenderer.cs index b4ee80663..8a3a82242 100644 --- a/OpenUtau.Core/Classic/ClassicRenderer.cs +++ b/OpenUtau.Core/Classic/ClassicRenderer.cs @@ -1,7 +1,10 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Numerics; +using System.Reflection; using System.Threading; using System.Threading.Tasks; using NAudio.Wave; @@ -13,6 +16,14 @@ using Serilog; namespace OpenUtau.Classic { + class MorphTrack { + public string TargetColor = null; + public string Flag = ""; + public string Abbr = ""; + public float[] Weights; + public int Hash = 17; + } + public class ClassicRenderer : IRenderer { static readonly HashSet supportedExp = new HashSet(){ Ustx.DYN, @@ -37,7 +48,8 @@ public class ClassicRenderer : IRenderer { public bool SupportsExpression(UExpressionDescriptor descriptor) { return descriptor.isFlag || !string.IsNullOrEmpty(descriptor.flag) - || supportedExp.Contains(descriptor.abbr); + || supportedExp.Contains(descriptor.abbr) + || descriptor.type == UExpressionType.MorphingCurve; } public RenderResult Layout(RenderPhrase phrase) { @@ -56,96 +68,1071 @@ public Task Render(RenderPhrase phrase, Progress progress, int tra } } - public Task RenderInternal(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender) { - var resamplerItems = new List(); - foreach (var phone in phrase.phones) { - resamplerItems.Add(new ResamplerItem(phrase, phone)); - } - var task = Task.Run(() => { - Parallel.ForEach(source: resamplerItems, parallelOptions: new ParallelOptions() { - MaxDegreeOfParallelism = Preferences.Default.NumRenderThreads - }, body: item => { - if (!cancellation.IsCancellationRequested && !File.Exists(item.outputFile)) { - if (!(item.resampler is WorldlineResampler)) { - VoicebankFiles.Inst.CopySourceTemp(item.inputFile, item.inputTemp); + // INTERNAL PHRASE LEVEL ENGINE + public Task RenderPhraseLevel(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender) { + return Task.Run(() => { + var result = Layout(phrase); + var wavtool = new SharpWavtool(true); + + var otoField = typeof(RenderPhone).GetField("oto", BindingFlags.Public | BindingFlags.Instance); + var baseOtos = new Dictionary(); + foreach (var phone in phrase.phones) baseOtos[phone] = phone.oto; + + if (TryGetMatrixData(phrase, trackNo, out string baseFlag, out List tracks, out var activePart, out int baseHash)) { + + var morphingFlags = new HashSet(); + foreach (var t in tracks) if (DocManager.Inst.Project.expressions.TryGetValue(t.Abbr, out var exp) && !string.IsNullOrEmpty(exp.flag)) morphingFlags.Add(exp.flag); + + var itemsBase = new List(); + var trackItems = new List>(); + for (int i = 0; i < tracks.Count; i++) trackItems.Add(new List()); + + var itemOtos = new ConcurrentDictionary(); + var singer = DocManager.Inst.Project.tracks[trackNo].Singer; + + foreach (var phone in phrase.phones) { + int relativeStartTick = (phrase.position + phone.position) - activePart.position; + int centerTick = relativeStartTick + (phone.duration / 2); + + otoField.SetValue(phone, baseOtos[phone]); + var itemBase = new ResamplerItem(phrase, phone); + + string staticFlags = ""; + if (itemBase.flags != null) { + var filteredFlags = itemBase.flags.Where(f => !morphingFlags.Any(mf => f.Item1.StartsWith(mf))); + staticFlags = string.Join("", filteredFlags.Select(f => f.Item1 + (f.Item2.HasValue ? f.Item2.Value.ToString() : ""))); } - if(!item.phone.direct){ - lock (Renderers.GetCacheLock(item.outputFile)) { - item.resampler.DoResamplerReturnsFile(item, Log.Logger); + + foreach (var c in activePart.curves) { + if (!c.IsEmpty && DocManager.Inst.Project.expressions.TryGetValue(c.abbr, out var exp) && !string.IsNullOrEmpty(exp.flag)) { + if (!tracks.Any(t => t.Abbr == exp.abbr) && !staticFlags.Contains(exp.flag)) staticFlags += $"{exp.flag}{(int)Math.Round((double)c.Sample(centerTick))}"; } + } + + int flagHashBase = HashString(staticFlags + baseFlag); + itemBase.flags = new[] { Tuple.Create(staticFlags + baseFlag, (int?)null, (string)null) }; + itemBase.outputFile = itemBase.outputFile.Replace(".wav", $"_pBase_{flagHashBase:X}.wav"); + itemsBase.Add(itemBase); + itemOtos[itemBase] = baseOtos[phone]; + + for (int i = 0; i < tracks.Count; i++) { + UOto targetOto = baseOtos[phone]; + if (!string.IsNullOrEmpty(tracks[i].TargetColor) && TryHijackOto(singer, phone, tracks[i].TargetColor, out var oto)) targetOto = oto; + + otoField.SetValue(phone, targetOto); + var tItem = new ResamplerItem(phrase, phone); + + int flagHashTrk = HashString(staticFlags + tracks[i].Flag); + tItem.flags = new[] { Tuple.Create(staticFlags + tracks[i].Flag, (int?)null, (string)null) }; + tItem.outputFile = tItem.outputFile.Replace(".wav", $"_pTrk{i}_{flagHashTrk:X}.wav"); + tItem.inputTemp = tItem.inputTemp.Replace(".wav", $"_hij_{flagHashTrk:X}.wav"); + + if (targetOto != baseOtos[phone]) tItem.inputFile = targetOto.File; + itemOtos[tItem] = targetOto; + trackItems[i].Add(tItem); + } + otoField.SetValue(phone, baseOtos[phone]); + } + + var allItems = new List(); + allItems.AddRange(itemsBase); + foreach (var list in trackItems) allItems.AddRange(list); + + int completed = 0; + int itemsPerPhone = allItems.Count / phrase.phones.Length; + if (itemsPerPhone == 0) itemsPerPhone = 1; + + Parallel.ForEach(allItems, new ParallelOptions() { MaxDegreeOfParallelism = Preferences.Default.NumRenderThreads }, item => { + if (cancellation.IsCancellationRequested) return; + lock (item.phone) { + otoField.SetValue(item.phone, itemOtos[item]); + if (!File.Exists(item.outputFile)) { - DocManager.Inst.Project.timeAxis.TickPosToBarBeat(item.phrase.position + item.phone.position, out int bar, out int beat, out int tick); - throw new InvalidDataException($"{item.resampler} failed to resample \"{item.phone.phoneme}\" at {bar}:{beat}.{string.Format("{0:000}", tick)}"); + if (!(item.resampler is WorldlineResampler)) VoicebankFiles.Inst.CopySourceTemp(item.inputFile, item.inputTemp); + lock (Renderers.GetCacheLock(item.outputFile)) item.resampler.DoResamplerReturnsFile(item, Log.Logger); + if (!(item.resampler is WorldlineResampler)) { + VoicebankFiles.Inst.CopyBackMetaFiles(item.inputFile, item.inputTemp); + try { + if (item.inputTemp.Contains("_hij_") && File.Exists(item.inputTemp)) { + File.Delete(item.inputTemp); + CleanUpMetaFiles(item.inputTemp); + } + } catch { } + } } + otoField.SetValue(item.phone, baseOtos[item.phone]); } - if (!(item.resampler is WorldlineResampler)) { - VoicebankFiles.Inst.CopyBackMetaFiles(item.inputFile, item.inputTemp); + if (Interlocked.Increment(ref completed) % itemsPerPhone == 0) progress.Complete(1, $"Track {trackNo + 1}: Morph: {item.resampler} \"{item.phone.phoneme}\""); + }); + + foreach (var item in itemsBase) otoField.SetValue(item.phone, itemOtos[item]); + float[] basePhraseSamples = wavtool.Concatenate(itemsBase, string.Empty, cancellation); + + var colorAudios = new List(); + var colorCurves = new List(); + + for (int i = 0; i < tracks.Count; i++) { + foreach (var item in trackItems[i]) otoField.SetValue(item.phone, itemOtos[item]); + float[] tSamples = wavtool.Concatenate(trackItems[i], string.Empty, cancellation); + colorAudios.Add(tSamples ?? basePhraseSamples); + colorCurves.Add(tracks[i].Weights); + } + + if (basePhraseSamples != null) result.samples = SpectralMorpher.MorphN(basePhraseSamples, colorAudios, colorCurves, 44100); + } else { + var standardItems = new List(); + var project = DocManager.Inst.Project; + var part = project.parts.OfType().FirstOrDefault(p => p.trackNo == trackNo && p.position <= phrase.position && p.End >= phrase.position); + + foreach (var phone in phrase.phones) { + var item = new ResamplerItem(phrase, phone); + + string staticFlags = item.flags != null ? string.Join("", item.flags.Select(f => f.Item1 + (f.Item2.HasValue ? f.Item2.Value.ToString() : ""))) : ""; + if (part != null) { + int relativeStartTick = (phrase.position + phone.position) - part.position; + int centerTick = relativeStartTick + (phone.duration / 2); + foreach (var c in part.curves) { + if (!c.IsEmpty && project.expressions.TryGetValue(c.abbr, out var exp) && !string.IsNullOrEmpty(exp.flag) && !staticFlags.Contains(exp.flag)) { + staticFlags += $"{exp.flag}{(int)Math.Round((double)c.Sample(centerTick))}"; + } + } } + + int flagHashStd = HashString(staticFlags); + item.flags = new[] { Tuple.Create(staticFlags, (int?)null, (string)null) }; + item.outputFile = item.outputFile.Replace(".wav", $"_std_{flagHashStd:X}.wav"); + standardItems.Add(item); } - progress.Complete(1, $"Track {trackNo + 1}: {item.resampler} \"{item.phone.phoneme}\""); - }); - var result = Layout(phrase); - var wavtool = new SharpWavtool(true); - result.samples = wavtool.Concatenate(resamplerItems, string.Empty, cancellation); - if (result.samples != null) { - Renderers.ApplyDynamics(phrase, result); + + Parallel.ForEach(standardItems, new ParallelOptions() { MaxDegreeOfParallelism = Preferences.Default.NumRenderThreads }, item => { + if (cancellation.IsCancellationRequested) return; + if (!File.Exists(item.outputFile)) { + if (!(item.resampler is WorldlineResampler)) VoicebankFiles.Inst.CopySourceTemp(item.inputFile, item.inputTemp); + lock (Renderers.GetCacheLock(item.outputFile)) item.resampler.DoResamplerReturnsFile(item, Log.Logger); + if (!(item.resampler is WorldlineResampler)) VoicebankFiles.Inst.CopyBackMetaFiles(item.inputFile, item.inputTemp); + } + progress.Complete(1, $"Track {trackNo + 1}: {item.resampler} \"{item.phone.phoneme}\""); + }); + result.samples = wavtool.Concatenate(standardItems, string.Empty, cancellation); } + + foreach (var phone in phrase.phones) otoField.SetValue(phone, baseOtos[phone]); + if (result.samples != null) Renderers.ApplyDynamics(phrase, result); return result; }); - return task; } - public Task RenderExternal(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender) { - var resamplerItems = new List(); - foreach (var phone in phrase.phones) { - resamplerItems.Add(new ResamplerItem(phrase, phone)); - } - var task = Task.Run(() => { - string progressInfo = $"Track {trackNo + 1} : {phrase.wavtool} \"{string.Join(" ", phrase.phones.Select(p => p.phoneme))}\""; - progress.Complete(0, progressInfo); - var wavPath = Path.Join(PathManager.Inst.CachePath, $"cat-{phrase.hash:x16}.wav"); + // EXTERNAL PHRASE LEVEL ENGINE + public Task RenderExternalPhraseLevel(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender) { + return Task.Run(() => { + int universalHash = GetUniversalCurveHash(phrase, trackNo); + var wavPath = Path.Join(PathManager.Inst.CachePath, $"cat-{phrase.hash:x16}_{universalHash:X}.wav"); phrase.AddCacheFile(wavPath); var result = Layout(phrase); if (File.Exists(wavPath)) { - try { - using (var waveStream = Wave.OpenFile(wavPath)) { - result.samples = Wave.GetSamples(waveStream.ToSampleProvider().ToMono(1, 0)); + try { using (var waveStream = Wave.OpenFile(wavPath)) { result.samples = Wave.GetSamples(waveStream.ToSampleProvider().ToMono(1, 0)); return result; } } catch { } + } + + var wavtool = ToolsManager.Inst.GetWavtool(phrase.wavtool); + var otoField = typeof(RenderPhone).GetField("oto", BindingFlags.Public | BindingFlags.Instance); + var baseOtos = new Dictionary(); + foreach (var phone in phrase.phones) baseOtos[phone] = phone.oto; + + if (TryGetMatrixData(phrase, trackNo, out string baseFlag, out List tracks, out var activePart, out int baseHash)) { + + var morphingFlags = new HashSet(); + foreach (var t in tracks) if (DocManager.Inst.Project.expressions.TryGetValue(t.Abbr, out var exp) && !string.IsNullOrEmpty(exp.flag)) morphingFlags.Add(exp.flag); + + var itemsBase = new List(); + var trackItems = new List>(); + for (int i = 0; i < tracks.Count; i++) trackItems.Add(new List()); + + var itemOtos = new Dictionary(); + var singer = DocManager.Inst.Project.tracks[trackNo].Singer; + + foreach (var phone in phrase.phones) { + int relativeStartTick = (phrase.position + phone.position) - activePart.position; + int centerTick = relativeStartTick + (phone.duration / 2); + + otoField.SetValue(phone, baseOtos[phone]); + var itemBase = new ResamplerItem(phrase, phone); + + string staticFlags = ""; + if (itemBase.flags != null) { + var filteredFlags = itemBase.flags.Where(f => !morphingFlags.Any(mf => f.Item1.StartsWith(mf))); + staticFlags = string.Join("", filteredFlags.Select(f => f.Item1 + (f.Item2.HasValue ? f.Item2.Value.ToString() : ""))); + } + + foreach (var c in activePart.curves) { + if (!c.IsEmpty && DocManager.Inst.Project.expressions.TryGetValue(c.abbr, out var exp) && !string.IsNullOrEmpty(exp.flag)) { + if (!tracks.Any(t => t.Abbr == exp.abbr) && !staticFlags.Contains(exp.flag)) staticFlags += $"{exp.flag}{(int)Math.Round((double)c.Sample(centerTick))}"; + } + } + + int flagHashBase = HashString(staticFlags + baseFlag); + itemBase.flags = new[] { Tuple.Create(staticFlags + baseFlag, (int?)null, (string)null) }; + itemBase.outputFile = itemBase.outputFile.Replace(".wav", $"_pBase_{flagHashBase:X}.wav"); + itemsBase.Add(itemBase); + itemOtos[itemBase] = baseOtos[phone]; + + for (int i = 0; i < tracks.Count; i++) { + UOto targetOto = baseOtos[phone]; + if (!string.IsNullOrEmpty(tracks[i].TargetColor) && TryHijackOto(singer, phone, tracks[i].TargetColor, out var oto)) targetOto = oto; + + otoField.SetValue(phone, targetOto); + var tItem = new ResamplerItem(phrase, phone); + + int flagHashTrk = HashString(staticFlags + tracks[i].Flag); + tItem.flags = new[] { Tuple.Create(staticFlags + tracks[i].Flag, (int?)null, (string)null) }; + tItem.outputFile = tItem.outputFile.Replace(".wav", $"_pTrk{i}_{flagHashTrk:X}.wav"); + tItem.inputTemp = tItem.inputTemp.Replace(".wav", $"_hij_{flagHashTrk:X}.wav"); + + if (targetOto != baseOtos[phone]) tItem.inputFile = targetOto.File; + itemOtos[tItem] = targetOto; + trackItems[i].Add(tItem); } - } catch (Exception e) { - Log.Error(e, $"Failed to render: failed to open {wavPath}"); + otoField.SetValue(phone, baseOtos[phone]); } - } - if (result.samples == null) { - foreach (var item in resamplerItems) { - VoicebankFiles.Inst.CopySourceTemp(item.inputFile, item.inputTemp); + + var allItems = new List(); + allItems.AddRange(itemsBase); + foreach (var list in trackItems) allItems.AddRange(list); + + Parallel.ForEach(allItems, new ParallelOptions() { MaxDegreeOfParallelism = Preferences.Default.NumRenderThreads }, item => { + if (cancellation.IsCancellationRequested) return; + lock (item.phone) { + otoField.SetValue(item.phone, itemOtos[item]); + if (!File.Exists(item.outputFile)) { + if (!(item.resampler is WorldlineResampler)) VoicebankFiles.Inst.CopySourceTemp(item.inputFile, item.inputTemp); + lock (Renderers.GetCacheLock(item.outputFile)) item.resampler.DoResamplerReturnsFile(item, Log.Logger); + if (!(item.resampler is WorldlineResampler)) { + VoicebankFiles.Inst.CopyBackMetaFiles(item.inputFile, item.inputTemp); + try { + if (item.inputTemp.Contains("_hij_") && File.Exists(item.inputTemp)) { + File.Delete(item.inputTemp); + CleanUpMetaFiles(item.inputTemp); + } + } catch { } + } + } + otoField.SetValue(item.phone, baseOtos[item.phone]); + } + progress.Complete(1, $"Track {trackNo + 1}: Morph: {phrase.wavtool} \"{item.phone.phoneme}\""); + }); + + string wavPathBase = wavPath.Replace(".wav", $"_base_{baseHash:X}.wav"); + + var colorPaths = new List(); + for (int i = 0; i < tracks.Count; i++) { + string p = wavPath.Replace(".wav", $"_trk{i}_{tracks[i].Hash:X}.wav"); + colorPaths.Add(p); } - var wavtool = ToolsManager.Inst.GetWavtool(phrase.wavtool); - result.samples = wavtool.Concatenate(resamplerItems, wavPath, cancellation); - foreach (var item in resamplerItems) { - VoicebankFiles.Inst.CopyBackMetaFiles(item.inputFile, item.inputTemp); + + foreach (var item in itemsBase) otoField.SetValue(item.phone, itemOtos[item]); + float[] basePhraseSamples = wavtool.Concatenate(itemsBase, wavPathBase, cancellation); + + // Fallback to read memory for external wavtools returning null + if (basePhraseSamples == null && File.Exists(wavPathBase)) { + basePhraseSamples = ReadWavToFloats(wavPathBase); } + + var colorAudios = new List(); + var colorCurves = new List(); + + for (int i = 0; i < tracks.Count; i++) { + foreach (var item in trackItems[i]) otoField.SetValue(item.phone, itemOtos[item]); + float[] tSamples = wavtool.Concatenate(trackItems[i], colorPaths[i], cancellation); + + // Fallback to read memory for external wavtools returning null + if (tSamples == null && File.Exists(colorPaths[i])) { + tSamples = ReadWavToFloats(colorPaths[i]); + } + + colorAudios.Add(tSamples ?? basePhraseSamples); + colorCurves.Add(tracks[i].Weights); + } + + if (basePhraseSamples != null) { + result.samples = SpectralMorpher.MorphN(basePhraseSamples, colorAudios, colorCurves, 44100); + WriteFloatsToWav(result.samples, wavPath); + + try { + if (File.Exists(wavPathBase)) { + File.Delete(wavPathBase); + CleanUpMetaFiles(wavPathBase); + } + foreach (var p in colorPaths) { + if (File.Exists(p)) { + File.Delete(p); + CleanUpMetaFiles(p); + } + } + } catch { } + } + } else { + var standardItems = new List(); + var project = DocManager.Inst.Project; + var part = project.parts.OfType().FirstOrDefault(p => p.trackNo == trackNo && p.position <= phrase.position && p.End >= phrase.position); + + foreach (var phone in phrase.phones) { + var item = new ResamplerItem(phrase, phone); + + string staticFlags = item.flags != null ? string.Join("", item.flags.Select(f => f.Item1 + (f.Item2.HasValue ? f.Item2.Value.ToString() : ""))) : ""; + if (part != null) { + int relativeStartTick = (phrase.position + phone.position) - part.position; + int centerTick = relativeStartTick + (phone.duration / 2); + foreach (var c in part.curves) { + if (!c.IsEmpty && project.expressions.TryGetValue(c.abbr, out var exp) && !string.IsNullOrEmpty(exp.flag) && !staticFlags.Contains(exp.flag)) { + staticFlags += $"{exp.flag}{(int)Math.Round((double)c.Sample(centerTick))}"; + } + } + } + + int flagHashStd = HashString(staticFlags); + item.flags = new[] { Tuple.Create(staticFlags, (int?)null, (string)null) }; + item.outputFile = item.outputFile.Replace(".wav", $"_std_{flagHashStd:X}.wav"); + standardItems.Add(item); + } + + foreach (var item in standardItems) VoicebankFiles.Inst.CopySourceTemp(item.inputFile, item.inputTemp); + result.samples = wavtool.Concatenate(standardItems, wavPath, cancellation); + foreach (var item in standardItems) VoicebankFiles.Inst.CopyBackMetaFiles(item.inputFile, item.inputTemp); + progress.Complete(phrase.phones.Length, $"Track {trackNo + 1} : {phrase.wavtool}"); } - progress.Complete(phrase.phones.Length, progressInfo); - if (result.samples != null) { - Renderers.ApplyDynamics(phrase, result); - } + + foreach (var phone in phrase.phones) otoField.SetValue(phone, baseOtos[phone]); + if (result.samples != null) Renderers.ApplyDynamics(phrase, result); return result; }); - return task; } - public RenderPitchResult LoadRenderedPitch(RenderPhrase phrase) { - return null; + // INTERNAL ALIAS LEVEL ENGINE + public Task RenderInternal(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender) { + if (Preferences.Default.PhraseLevelMorphing) return RenderPhraseLevel(phrase, progress, trackNo, cancellation, isPreRender); + + return Task.Run(() => { + var otoField = typeof(RenderPhone).GetField("oto", BindingFlags.Public | BindingFlags.Instance); + var baseOtos = new Dictionary(); + foreach (var phone in phrase.phones) baseOtos[phone] = phone.oto; + + var finalItems = new List(); + + Parallel.ForEach(phrase.phones, new ParallelOptions() { MaxDegreeOfParallelism = Preferences.Default.NumRenderThreads }, phone => { + if (cancellation.IsCancellationRequested) return; + + if (TryGetMatrixDataAlias(phrase, phone, trackNo, out string baseFlag, out List tracks, out var activePart, out int baseHash)) { + + var morphingFlags = new HashSet(); + foreach (var t in tracks) if (DocManager.Inst.Project.expressions.TryGetValue(t.Abbr, out var exp) && !string.IsNullOrEmpty(exp.flag)) morphingFlags.Add(exp.flag); + + int relativeStartTick = (phrase.position + phone.position) - activePart.position; + int centerTick = relativeStartTick + (phone.duration / 2); + + otoField.SetValue(phone, baseOtos[phone]); + var itemBase = new ResamplerItem(phrase, phone); + string staticFlags = ""; + if (itemBase.flags != null) { + var filteredFlags = itemBase.flags.Where(f => !morphingFlags.Any(mf => f.Item1.StartsWith(mf))); + staticFlags = string.Join("", filteredFlags.Select(f => f.Item1 + (f.Item2.HasValue ? f.Item2.Value.ToString() : ""))); + } + foreach (var c in activePart.curves) { + if (!c.IsEmpty && DocManager.Inst.Project.expressions.TryGetValue(c.abbr, out var exp) && !string.IsNullOrEmpty(exp.flag)) { + if (!tracks.Any(t => t.Abbr == exp.abbr) && !staticFlags.Contains(exp.flag)) staticFlags += $"{exp.flag}{(int)Math.Round((double)c.Sample(centerTick))}"; + } + } + + itemBase.flags = new[] { Tuple.Create(staticFlags + baseFlag, (int?)null, (string)null) }; + string masterMorphPath = itemBase.outputFile.Replace(".wav", $"_morphN_{baseHash:X}.wav"); + lock (itemBase.phrase) { itemBase.phrase.AddCacheFile(masterMorphPath); } + + var masterItem = new ResamplerItem(phrase, phone) { outputFile = masterMorphPath }; + lock(finalItems) { finalItems.Add(masterItem); } + + if (!File.Exists(masterMorphPath)) { + string fileBase = masterMorphPath + ".base.wav"; + RunResamplerWithFlag(itemBase, staticFlags + baseFlag, fileBase); + + var colorFiles = new List(); + var singer = DocManager.Inst.Project.tracks[trackNo].Singer; + + for (int i = 0; i < tracks.Count; i++) { + string fileTrk = masterMorphPath + $".trk{i}.wav"; + colorFiles.Add(fileTrk); + + UOto targetOto = baseOtos[phone]; + if (!string.IsNullOrEmpty(tracks[i].TargetColor) && TryHijackOto(singer, phone, tracks[i].TargetColor, out var oto)) targetOto = oto; + + otoField.SetValue(phone, targetOto); + var tItem = new ResamplerItem(phrase, phone); + tItem.flags = new[] { Tuple.Create(staticFlags + tracks[i].Flag, (int?)null, (string)null) }; + tItem.inputTemp = tItem.inputTemp.Replace(".wav", $"_hij_{tracks[i].Hash:X}.wav"); + if (targetOto != baseOtos[phone]) tItem.inputFile = targetOto.File; + + RunResamplerWithFlag(tItem, staticFlags + tracks[i].Flag, fileTrk); + } + otoField.SetValue(phone, baseOtos[phone]); + + if (File.Exists(fileBase)) { + float[] baseSamples = ReadWavToFloats(fileBase); + var colorAudios = new List(); + var colorCurves = new List(); + + for (int i = 0; i < tracks.Count; i++) { + colorAudios.Add(File.Exists(colorFiles[i]) ? ReadWavToFloats(colorFiles[i]) : baseSamples); + colorCurves.Add(tracks[i].Weights); + } + WriteFloatsToWav(SpectralMorpher.MorphN(baseSamples, colorAudios, colorCurves, 44100), masterMorphPath); + + try { + if (File.Exists(fileBase)) { + File.Delete(fileBase); + CleanUpMetaFiles(fileBase); + } + foreach (var f in colorFiles) { + if (File.Exists(f)) { + File.Delete(f); + CleanUpMetaFiles(f); + } + } + } catch { } + } + } + progress.Complete(1, $"Track {trackNo + 1}: Morph: {itemBase.resampler} \"{phone.phoneme}\""); + } else { + otoField.SetValue(phone, baseOtos[phone]); + var item = new ResamplerItem(phrase, phone); + + var project = DocManager.Inst.Project; + int phoneAbsoluteTick = phrase.position + phone.position; + var part = project.parts.OfType().FirstOrDefault(p => p.trackNo == trackNo && p.position <= phoneAbsoluteTick && p.End >= phoneAbsoluteTick); + string staticFlags = item.flags != null ? string.Join("", item.flags.Select(f => f.Item1 + (f.Item2.HasValue ? f.Item2.Value.ToString() : ""))) : ""; + + if (part != null) { + int relativeStartTick = phoneAbsoluteTick - part.position; + int centerTick = relativeStartTick + (phone.duration / 2); + foreach (var c in part.curves) { + if (!c.IsEmpty && project.expressions.TryGetValue(c.abbr, out var exp) && !string.IsNullOrEmpty(exp.flag) && !staticFlags.Contains(exp.flag)) { + staticFlags += $"{exp.flag}{(int)Math.Round((double)c.Sample(centerTick))}"; + } + } + } + + int flagHash = HashString(staticFlags); + item.flags = new[] { Tuple.Create(staticFlags, (int?)null, (string)null) }; + item.outputFile = item.outputFile.Replace(".wav", $"_std_{flagHash:X}.wav"); + + lock(finalItems) { finalItems.Add(item); } + + if (!File.Exists(item.outputFile)) { + if (!(item.resampler is WorldlineResampler)) VoicebankFiles.Inst.CopySourceTemp(item.inputFile, item.inputTemp); + if(!item.phone.direct) lock (Renderers.GetCacheLock(item.outputFile)) item.resampler.DoResamplerReturnsFile(item, Log.Logger); + if (!(item.resampler is WorldlineResampler)) VoicebankFiles.Inst.CopyBackMetaFiles(item.inputFile, item.inputTemp); + } + progress.Complete(1, $"Track {trackNo + 1}: {item.resampler} \"{phone.phoneme}\""); + } + }); + + foreach (var phone in phrase.phones) otoField.SetValue(phone, baseOtos[phone]); + finalItems = finalItems.OrderBy(i => i.phone.position).ToList(); + + var result = Layout(phrase); + result.samples = new SharpWavtool(true).Concatenate(finalItems, string.Empty, cancellation); + if (result.samples != null) Renderers.ApplyDynamics(phrase, result); + return result; + }); } + // EXTERNAL ALIAS LEVEL ENGINE + public Task RenderExternal(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender) { + if (Preferences.Default.PhraseLevelMorphing) return RenderExternalPhraseLevel(phrase, progress, trackNo, cancellation, isPreRender); + + int universalHash = GetUniversalCurveHash(phrase, trackNo); + return Task.Run(() => { + var wavPath = Path.Join(PathManager.Inst.CachePath, $"cat-{phrase.hash:x16}_{universalHash:X}.wav"); + phrase.AddCacheFile(wavPath); + + var result = Layout(phrase); + if (File.Exists(wavPath)) { + try { using (var waveStream = Wave.OpenFile(wavPath)) { result.samples = Wave.GetSamples(waveStream.ToSampleProvider().ToMono(1, 0)); return result; } } catch { } + } + + var otoField = typeof(RenderPhone).GetField("oto", BindingFlags.Public | BindingFlags.Instance); + var baseOtos = new Dictionary(); + foreach (var phone in phrase.phones) baseOtos[phone] = phone.oto; + + var finalItems = new List(); + + Parallel.ForEach(phrase.phones, new ParallelOptions() { MaxDegreeOfParallelism = Preferences.Default.NumRenderThreads }, phone => { + if (cancellation.IsCancellationRequested) return; + + if (TryGetMatrixDataAlias(phrase, phone, trackNo, out string baseFlag, out List tracks, out var activePart, out int baseHash)) { + + var morphingFlags = new HashSet(); + foreach (var t in tracks) if (DocManager.Inst.Project.expressions.TryGetValue(t.Abbr, out var exp) && !string.IsNullOrEmpty(exp.flag)) morphingFlags.Add(exp.flag); + + int relativeStartTick = (phrase.position + phone.position) - activePart.position; + int centerTick = relativeStartTick + (phone.duration / 2); + + otoField.SetValue(phone, baseOtos[phone]); + var itemBase = new ResamplerItem(phrase, phone); + string staticFlags = ""; + if (itemBase.flags != null) { + var filteredFlags = itemBase.flags.Where(f => !morphingFlags.Any(mf => f.Item1.StartsWith(mf))); + staticFlags = string.Join("", filteredFlags.Select(f => f.Item1 + (f.Item2.HasValue ? f.Item2.Value.ToString() : ""))); + } + foreach (var c in activePart.curves) { + if (!c.IsEmpty && DocManager.Inst.Project.expressions.TryGetValue(c.abbr, out var exp) && !string.IsNullOrEmpty(exp.flag)) { + if (!tracks.Any(t => t.Abbr == exp.abbr) && !staticFlags.Contains(exp.flag)) staticFlags += $"{exp.flag}{(int)Math.Round((double)c.Sample(centerTick))}"; + } + } + + itemBase.flags = new[] { Tuple.Create(staticFlags + baseFlag, (int?)null, (string)null) }; + string masterMorphPath = itemBase.outputFile.Replace(".wav", $"_morphN_{baseHash:X}.wav"); + lock (itemBase.phrase) { itemBase.phrase.AddCacheFile(masterMorphPath); } + + var masterItem = new ResamplerItem(phrase, phone) { outputFile = masterMorphPath }; + lock(finalItems) { finalItems.Add(masterItem); } + + if (!File.Exists(masterMorphPath)) { + string fileBase = masterMorphPath + ".base.wav"; + RunResamplerWithFlag(itemBase, staticFlags + baseFlag, fileBase); + + var colorFiles = new List(); + var singer = DocManager.Inst.Project.tracks[trackNo].Singer; + + for (int i = 0; i < tracks.Count; i++) { + string fileTrk = masterMorphPath + $".trk{i}.wav"; + colorFiles.Add(fileTrk); + + UOto targetOto = baseOtos[phone]; + if (!string.IsNullOrEmpty(tracks[i].TargetColor) && TryHijackOto(singer, phone, tracks[i].TargetColor, out var oto)) targetOto = oto; + + otoField.SetValue(phone, targetOto); + var tItem = new ResamplerItem(phrase, phone); + tItem.flags = new[] { Tuple.Create(staticFlags + tracks[i].Flag, (int?)null, (string)null) }; + tItem.inputTemp = tItem.inputTemp.Replace(".wav", $"_hij_{tracks[i].Hash:X}.wav"); + if (targetOto != baseOtos[phone]) tItem.inputFile = targetOto.File; + + RunResamplerWithFlag(tItem, staticFlags + tracks[i].Flag, fileTrk); + } + otoField.SetValue(phone, baseOtos[phone]); + + if (File.Exists(fileBase)) { + float[] baseSamples = ReadWavToFloats(fileBase); + var colorAudios = new List(); + var colorCurves = new List(); + + for (int i = 0; i < tracks.Count; i++) { + colorAudios.Add(File.Exists(colorFiles[i]) ? ReadWavToFloats(colorFiles[i]) : baseSamples); + colorCurves.Add(tracks[i].Weights); + } + WriteFloatsToWav(SpectralMorpher.MorphN(baseSamples, colorAudios, colorCurves, 44100), masterMorphPath); + + try { + try { + if (File.Exists(fileBase)) { + File.Delete(fileBase); + CleanUpMetaFiles(fileBase); + } + foreach (var f in colorFiles) { + if (File.Exists(f)) { + File.Delete(f); + CleanUpMetaFiles(f); + } + } + } catch { } + foreach (var f in colorFiles) if (File.Exists(f)) File.Delete(f); + } catch { } + } + } + progress.Complete(1, $"Track {trackNo + 1}: Morph: {phrase.wavtool} \"{phone.phoneme}\""); + } else { + otoField.SetValue(phone, baseOtos[phone]); + var item = new ResamplerItem(phrase, phone); + + var project = DocManager.Inst.Project; + int phoneAbsoluteTick = phrase.position + phone.position; + var part = project.parts.OfType().FirstOrDefault(p => p.trackNo == trackNo && p.position <= phoneAbsoluteTick && p.End >= phoneAbsoluteTick); + string staticFlags = item.flags != null ? string.Join("", item.flags.Select(f => f.Item1 + (f.Item2.HasValue ? f.Item2.Value.ToString() : ""))) : ""; + + if (part != null) { + int relativeStartTick = phoneAbsoluteTick - part.position; + int centerTick = relativeStartTick + (phone.duration / 2); + foreach (var c in part.curves) { + if (!c.IsEmpty && project.expressions.TryGetValue(c.abbr, out var exp) && !string.IsNullOrEmpty(exp.flag) && !staticFlags.Contains(exp.flag)) { + staticFlags += $"{exp.flag}{(int)Math.Round((double)c.Sample(centerTick))}"; + } + } + } + + int flagHash = HashString(staticFlags); + item.flags = new[] { Tuple.Create(staticFlags, (int?)null, (string)null) }; + item.outputFile = item.outputFile.Replace(".wav", $"_std_{flagHash:X}.wav"); + + lock(finalItems) { finalItems.Add(item); } + + if (!File.Exists(item.outputFile)) { + if (!(item.resampler is WorldlineResampler)) VoicebankFiles.Inst.CopySourceTemp(item.inputFile, item.inputTemp); + if(!item.phone.direct) lock (Renderers.GetCacheLock(item.outputFile)) item.resampler.DoResamplerReturnsFile(item, Log.Logger); + if (!(item.resampler is WorldlineResampler)) VoicebankFiles.Inst.CopyBackMetaFiles(item.inputFile, item.inputTemp); + } + progress.Complete(1, $"Track {trackNo + 1}: {phrase.wavtool} \"{phone.phoneme}\""); + } + }); + + foreach (var phone in phrase.phones) otoField.SetValue(phone, baseOtos[phone]); + finalItems = finalItems.OrderBy(i => i.phone.position).ToList(); + result.samples = ToolsManager.Inst.GetWavtool(phrase.wavtool).Concatenate(finalItems, wavPath, cancellation); + + if (result.samples != null) Renderers.ApplyDynamics(phrase, result); + return result; + }); + } + + public RenderPitchResult LoadRenderedPitch(RenderPhrase phrase) { return null; } + public UExpressionDescriptor[] GetSuggestedExpressions(USinger singer, URenderSettings renderSettings) { - var manifest= renderSettings.Resampler.Manifest; - if (manifest == null) { - return new UExpressionDescriptor[] { }; + var expressions = new List(); + var manifest = renderSettings.Resampler.Manifest; + if (manifest != null && manifest.expressions != null) expressions.AddRange(manifest.expressions.Values); + + if (singer != null && singer.Subbanks != null) { + var uniqueColors = new HashSet(); + foreach (var sub in singer.Subbanks) if (!string.IsNullOrEmpty(sub.Color)) uniqueColors.Add(sub.Color); + int colorIndex = 1; + foreach (var colorName in uniqueColors) { + expressions.Add(new UExpressionDescriptor { name = $"voice color {colorIndex:D2} {colorName}", abbr = $"cl{colorIndex:D2}", type = UExpressionType.MorphingCurve, min = 0, max = 100, defaultValue = 0, isFlag = false, flag = "" }); + colorIndex++; + } } - return manifest.expressions.Values.ToArray(); + return expressions.ToArray(); } public override string ToString() => Renderers.CLASSIC; + + private bool TryHijackOto(USinger singer, RenderPhone phone, string targetColor, out UOto targetOto) { + targetOto = null; + if (singer == null || singer.Subbanks == null) return false; + + var targetSubbanks = singer.Subbanks.Where(b => b.Color == targetColor).ToList(); + if (targetSubbanks.Count == 0) return false; + + if (singer.TryGetMappedOto(phone.phoneme, phone.tone, targetColor, out targetOto)) { + bool isCorrectColor = false; + foreach(var tb in targetSubbanks) { + if ((string.IsNullOrEmpty(tb.Prefix) || targetOto.Alias.StartsWith(tb.Prefix)) && + (string.IsNullOrEmpty(tb.Suffix) || targetOto.Alias.EndsWith(tb.Suffix))) { + isCorrectColor = true; break; + } + } + if (isCorrectColor) return true; + } + + string rawAlias = phone.oto.Alias; + var baseSubbank = singer.Subbanks.OrderByDescending(b => (b.Prefix?.Length ?? 0) + (b.Suffix?.Length ?? 0)) + .FirstOrDefault(b => (string.IsNullOrEmpty(b.Prefix) || rawAlias.StartsWith(b.Prefix)) && + (string.IsNullOrEmpty(b.Suffix) || rawAlias.EndsWith(b.Suffix))); + + string strippedAlias = rawAlias; + if (baseSubbank != null) { + if (!string.IsNullOrEmpty(baseSubbank.Prefix) && strippedAlias.StartsWith(baseSubbank.Prefix)) strippedAlias = strippedAlias.Substring(baseSubbank.Prefix.Length); + if (!string.IsNullOrEmpty(baseSubbank.Suffix) && strippedAlias.EndsWith(baseSubbank.Suffix)) strippedAlias = strippedAlias.Substring(0, strippedAlias.Length - baseSubbank.Suffix.Length); + foreach(var tb in targetSubbanks) if (singer.TryGetOto(tb.Prefix + strippedAlias + tb.Suffix, out targetOto)) return true; + } + + string rawPhoneme = phone.phoneme; + if (baseSubbank != null) { + if (!string.IsNullOrEmpty(baseSubbank.Prefix) && rawPhoneme.StartsWith(baseSubbank.Prefix)) rawPhoneme = rawPhoneme.Substring(baseSubbank.Prefix.Length); + if (!string.IsNullOrEmpty(baseSubbank.Suffix) && rawPhoneme.EndsWith(baseSubbank.Suffix)) rawPhoneme = rawPhoneme.Substring(0, rawPhoneme.Length - baseSubbank.Suffix.Length); + } + + var bestSubbank = targetSubbanks.OrderBy(b => { + if (b.toneSet == null || b.toneSet.Count == 0) return 999; + return Math.Abs(b.toneSet.FirstOrDefault() - phone.tone); + }).FirstOrDefault(); + + if (bestSubbank != null) { + if (singer.TryGetOto(bestSubbank.Prefix + rawPhoneme + bestSubbank.Suffix, out targetOto)) return true; + if (singer.TryGetOto(bestSubbank.Prefix + strippedAlias + bestSubbank.Suffix, out targetOto)) return true; + } + + foreach(var tb in targetSubbanks) if (singer.TryGetOto(tb.Prefix + rawPhoneme + tb.Suffix, out targetOto)) return true; + return false; + } + + private float[] ReadWavToFloats(string filePath) { + using (var waveStream = Wave.OpenFile(filePath)) { return Wave.GetSamples(waveStream.ToSampleProvider().ToMono(1, 0)); } + } + + private void WriteFloatsToWav(float[] samples, string filePath) { + var waveFormat = new NAudio.Wave.WaveFormat(44100, 16, 1); + using (var writer = new NAudio.Wave.WaveFileWriter(filePath, waveFormat)) { + short[] shortSamples = new short[samples.Length]; + for (int i = 0; i < samples.Length; i++) { + float sample = samples[i] * 32768f; + if (sample > 32767f) sample = 32767f; + if (sample < -32768f) sample = -32768f; + shortSamples[i] = (short)sample; + } + byte[] byteBuffer = new byte[shortSamples.Length * 2]; + Buffer.BlockCopy(shortSamples, 0, byteBuffer, 0, byteBuffer.Length); + writer.Write(byteBuffer, 0, byteBuffer.Length); + } + } + + private void RunResamplerWithFlag(ResamplerItem baseItem, string tempFlag, string tempOutputPath) { + if (!(baseItem.resampler is WorldlineResampler)) VoicebankFiles.Inst.CopySourceTemp(baseItem.inputFile, baseItem.inputTemp); + var originalFlags = baseItem.flags; + string originalOutput = baseItem.outputFile; + try { + baseItem.flags = new[] { Tuple.Create(tempFlag, (int?)null, (string)null) }; + baseItem.outputFile = tempOutputPath; + lock (Renderers.GetCacheLock(tempOutputPath)) baseItem.resampler.DoResamplerReturnsFile(baseItem, Log.Logger); + } finally { + baseItem.flags = originalFlags; + baseItem.outputFile = originalOutput; + if (!(baseItem.resampler is WorldlineResampler)) { + VoicebankFiles.Inst.CopyBackMetaFiles(baseItem.inputFile, baseItem.inputTemp); + try { + if (baseItem.inputTemp.Contains("_hij_") && File.Exists(baseItem.inputTemp)) { + File.Delete(baseItem.inputTemp); + CleanUpMetaFiles(baseItem.inputTemp); + } + } catch { } + } + } + } + + private int HashString(string str) { + if (string.IsNullOrEmpty(str)) return 0; + unchecked { + int hash = 23; + foreach (char c in str) hash = hash * 31 + c; + return hash; + } + } + + private int GetUniversalCurveHash(RenderPhrase phrase, int trackNo) { + int hash = 17; + var part = DocManager.Inst.Project.parts.OfType().FirstOrDefault(p => p.trackNo == trackNo && p.position <= phrase.position && p.End >= phrase.position); + if (part != null) { + int startTick = phrase.position - part.position; + int endTick = startTick + phrase.duration; + for (int tick = startTick; tick <= endTick; tick += 15) { + foreach (var c in part.curves) if (!c.IsEmpty) { + hash = unchecked(hash * 31 + HashString(c.abbr)); + hash = unchecked(hash * 31 + (int)Math.Round((double)c.Sample(tick))); + } + } + } + return hash; + } + + private bool TryGetMatrixData(RenderPhrase phrase, int trackNo, out string baseFlag, out List tracks, out UVoicePart activePart, out int baseHash) { + baseFlag = ""; tracks = new List(); baseHash = 17; + var project = DocManager.Inst.Project; + activePart = project.parts.OfType().FirstOrDefault(p => p.trackNo == trackNo && p.position <= phrase.position && p.End >= phrase.position); + if (activePart == null) return false; + + int startTick = phrase.position - activePart.position; + int endTick = startTick + phrase.duration; + double phraseStartMs = phrase.positionMs - phrase.leadingMs; + int sampleCount = (int)((phrase.positionMs + phrase.durationMs + 200.0 - phraseStartMs) / 5.0) + 1; + + var morphingExps = project.expressions.Values.Where(exp => exp.type == UExpressionType.MorphingCurve); + var dynamicExps = new List<(UExpressionDescriptor exp, UCurve c, float cMin, float cMax, string color)>(); + + foreach (var exp in morphingExps) { + var c = activePart.curves.FirstOrDefault(x => x.abbr == exp.abbr); + if (c == null || c.IsEmpty) continue; + + float cMin = float.MaxValue, cMax = float.MinValue; + for (int tick = startTick; tick <= endTick; tick += 15) { + float v = c.Sample(tick); + if (v < cMin) cMin = v; + if (v > cMax) cMax = v; + } + + string matchedColor = null; + var singer = project.tracks[trackNo].Singer; + if (singer != null && singer.Subbanks != null) { + foreach (var sub in singer.Subbanks) if (!string.IsNullOrEmpty(sub.Color) && exp.name.IndexOf(sub.Color, StringComparison.OrdinalIgnoreCase) >= 0) matchedColor = sub.Color; + } + + if (matchedColor != null) { + if (cMax > 0.05f) dynamicExps.Add((exp, c, cMin, cMax, matchedColor)); + } else { + float defVal = exp.defaultValue; + if (cMax > defVal + 0.1f || cMin < defVal - 0.1f) { + dynamicExps.Add((exp, c, cMin, cMax, null)); + string resFlag = string.IsNullOrEmpty(exp.flag) ? exp.abbr : exp.flag; + baseFlag += $"{resFlag}{(int)Math.Round((double)defVal)}"; + baseHash = unchecked(baseHash * 31 + HashString(exp.abbr)); + } + } + } + + // Protect consonant phase integrity + bool[] isVoiced = new bool[sampleCount]; + for (int i = 0; i < sampleCount; i++) isVoiced[i] = true; + foreach (var phone in phrase.phones) { + double startMs = phone.positionMs - phone.leadingMs; + double preutterMs = phone.positionMs; + int startIdx = Math.Max(0, (int)((startMs - phraseStartMs) / 5.0)); + int endIdx = Math.Min(sampleCount - 1, (int)((preutterMs - phraseStartMs) / 5.0)); + for (int i = startIdx; i <= endIdx; i++) isVoiced[i] = false; + } + + foreach (var data in dynamicExps) { + if (data.color != null) { + float[] rawWeights = new float[sampleCount]; + float[] finalWeights = new float[sampleCount]; + int trackHash = 17; + trackHash = unchecked(trackHash * 31 + HashString(data.color)); + + for (int i = 0; i < sampleCount; i++) { + int tick = project.timeAxis.MsPosToTickPos(phraseStartMs + (i * 5.0)) - activePart.position; + float val = Math.Clamp(data.c.Sample(tick), 0, 100); + rawWeights[i] = isVoiced[i] ? val : (val > 50f ? 100f : 0f); + } + + // 5-tap moving average to stop STFT cliff popping + for (int i = 0; i < sampleCount; i++) { + if (i < 2 || i >= sampleCount - 2) finalWeights[i] = rawWeights[i]; + else finalWeights[i] = (rawWeights[i - 2] + rawWeights[i - 1] + rawWeights[i] + rawWeights[i + 1] + rawWeights[i + 2]) / 5f; + trackHash = unchecked(trackHash * 31 + (int)Math.Round(finalWeights[i])); + } + + baseHash = unchecked(baseHash * 31 + trackHash); + tracks.Add(new MorphTrack { TargetColor = data.color, Abbr = data.exp.abbr, Flag = baseFlag, Weights = finalWeights, Hash = trackHash }); + } else { + float defVal = data.exp.defaultValue; + + if (data.cMax > defVal + 0.1f) { + float[] rawWeights = new float[sampleCount]; + float[] finalWeights = new float[sampleCount]; + int trackHash = 17; + trackHash = unchecked(trackHash * 31 + HashString(data.exp.abbr)); + + for (int i = 0; i < sampleCount; i++) { + int tick = project.timeAxis.MsPosToTickPos(phraseStartMs + (i * 5.0)) - activePart.position; + float val = data.c.Sample(tick); + float rw = val > defVal ? Math.Clamp(((val - defVal) / Math.Max(0.001f, data.exp.max - defVal)) * 100f, 0, 100) : 0; + rawWeights[i] = isVoiced[i] ? rw : (rw > 50f ? 100f : 0f); + } + + for (int i = 0; i < sampleCount; i++) { + if (i < 2 || i >= sampleCount - 2) finalWeights[i] = rawWeights[i]; + else finalWeights[i] = (rawWeights[i - 2] + rawWeights[i - 1] + rawWeights[i] + rawWeights[i + 1] + rawWeights[i + 2]) / 5f; + trackHash = unchecked(trackHash * 31 + (int)Math.Round(finalWeights[i])); + } + + string trkFlag = ""; + foreach(var d2 in dynamicExps) { + if (d2.color != null) continue; + string r2 = string.IsNullOrEmpty(d2.exp.flag) ? d2.exp.abbr : d2.exp.flag; + trkFlag += (d2.exp.abbr == data.exp.abbr) ? $"{r2}{(int)Math.Round((double)data.cMax)}" : $"{r2}{(int)Math.Round((double)d2.exp.defaultValue)}"; + } + baseHash = unchecked(baseHash * 31 + trackHash); + tracks.Add(new MorphTrack { TargetColor = null, Abbr = data.exp.abbr, Flag = trkFlag, Weights = finalWeights, Hash = trackHash }); + } + if (data.cMin < defVal - 0.1f) { + float[] rawWeights = new float[sampleCount]; + float[] finalWeights = new float[sampleCount]; + int trackHash = 17; + trackHash = unchecked(trackHash * 31 + HashString(data.exp.abbr)); + + for (int i = 0; i < sampleCount; i++) { + int tick = project.timeAxis.MsPosToTickPos(phraseStartMs + (i * 5.0)) - activePart.position; + float val = data.c.Sample(tick); + float rw = val < defVal ? Math.Clamp(((defVal - val) / Math.Max(0.001f, defVal - data.exp.min)) * 100f, 0, 100) : 0; + rawWeights[i] = isVoiced[i] ? rw : (rw > 50f ? 100f : 0f); + } + + for (int i = 0; i < sampleCount; i++) { + if (i < 2 || i >= sampleCount - 2) finalWeights[i] = rawWeights[i]; + else finalWeights[i] = (rawWeights[i - 2] + rawWeights[i - 1] + rawWeights[i] + rawWeights[i + 1] + rawWeights[i + 2]) / 5f; + trackHash = unchecked(trackHash * 31 + (int)Math.Round(finalWeights[i])); + } + + string trkFlag = ""; + foreach(var d2 in dynamicExps) { + if (d2.color != null) continue; + string r2 = string.IsNullOrEmpty(d2.exp.flag) ? d2.exp.abbr : d2.exp.flag; + trkFlag += (d2.exp.abbr == data.exp.abbr) ? $"{r2}{(int)Math.Round((double)data.cMin)}" : $"{r2}{(int)Math.Round((double)d2.exp.defaultValue)}"; + } + baseHash = unchecked(baseHash * 31 + trackHash); + tracks.Add(new MorphTrack { TargetColor = null, Abbr = data.exp.abbr, Flag = trkFlag, Weights = finalWeights, Hash = trackHash }); + } + } + } + return tracks.Count > 0; + } + + private bool TryGetMatrixDataAlias(RenderPhrase phrase, RenderPhone phone, int trackNo, out string baseFlag, out List tracks, out UVoicePart activePart, out int baseHash) { + baseFlag = ""; tracks = new List(); baseHash = 17; + var project = DocManager.Inst.Project; + int phoneAbsoluteTick = phrase.position + phone.position; + activePart = project.parts.OfType().FirstOrDefault(p => p.trackNo == trackNo && p.position <= phoneAbsoluteTick && p.End >= phoneAbsoluteTick); + if (activePart == null) return false; + + int startTick = (phrase.position + phone.position) - activePart.position; + int endTick = startTick + phone.duration; + double phonePosMs = phone.positionMs - phone.leadingMs; + int sampleCount = (int)((phone.endMs - phonePosMs) / 5.0) + 1; + + var morphingExps = project.expressions.Values.Where(exp => exp.type == UExpressionType.MorphingCurve); + var dynamicExps = new List<(UExpressionDescriptor exp, UCurve c, float cMin, float cMax, string color)>(); + + foreach (var exp in morphingExps) { + var c = activePart.curves.FirstOrDefault(x => x.abbr == exp.abbr); + if (c == null || c.IsEmpty) continue; + + float cMin = float.MaxValue, cMax = float.MinValue; + for (int tick = startTick; tick <= endTick; tick += 15) { + float v = c.Sample(tick); + if (v < cMin) cMin = v; + if (v > cMax) cMax = v; + } + + string matchedColor = null; + var singer = project.tracks[trackNo].Singer; + if (singer != null && singer.Subbanks != null) { + foreach (var sub in singer.Subbanks) if (!string.IsNullOrEmpty(sub.Color) && exp.name.IndexOf(sub.Color, StringComparison.OrdinalIgnoreCase) >= 0) matchedColor = sub.Color; + } + + if (matchedColor != null) { + if (cMax > 0.05f) dynamicExps.Add((exp, c, cMin, cMax, matchedColor)); + } else { + float defVal = exp.defaultValue; + if (cMax > defVal + 0.1f || cMin < defVal - 0.1f) { + dynamicExps.Add((exp, c, cMin, cMax, null)); + string resFlag = string.IsNullOrEmpty(exp.flag) ? exp.abbr : exp.flag; + baseFlag += $"{resFlag}{(int)Math.Round((double)defVal)}"; + baseHash = unchecked(baseHash * 31 + HashString(exp.abbr)); + } + } + } + + // isVoiced for Alias level + bool[] isVoiced = new bool[sampleCount]; + for (int i = 0; i < sampleCount; i++) isVoiced[i] = true; + double startMs = phone.positionMs - phone.leadingMs; + double preutterMs = phone.positionMs; + int startIdx = Math.Max(0, (int)((startMs - phonePosMs) / 5.0)); + int endIdx = Math.Min(sampleCount - 1, (int)((preutterMs - phonePosMs) / 5.0)); + for (int i = startIdx; i <= endIdx; i++) isVoiced[i] = false; + + foreach (var data in dynamicExps) { + if (data.color != null) { + float[] rawWeights = new float[sampleCount]; + float[] finalWeights = new float[sampleCount]; + int trackHash = 17; + trackHash = unchecked(trackHash * 31 + HashString(data.color)); + + for (int i = 0; i < sampleCount; i++) { + int tick = project.timeAxis.MsPosToTickPos(phonePosMs + (i * 5.0)) - activePart.position; + float val = Math.Clamp(data.c.Sample(tick), 0, 100); + rawWeights[i] = isVoiced[i] ? val : (val > 50f ? 100f : 0f); + } + + for (int i = 0; i < sampleCount; i++) { + if (i < 2 || i >= sampleCount - 2) finalWeights[i] = rawWeights[i]; + else finalWeights[i] = (rawWeights[i - 2] + rawWeights[i - 1] + rawWeights[i] + rawWeights[i + 1] + rawWeights[i + 2]) / 5f; + trackHash = unchecked(trackHash * 31 + (int)Math.Round(finalWeights[i])); + } + + baseHash = unchecked(baseHash * 31 + trackHash); + tracks.Add(new MorphTrack { TargetColor = data.color, Abbr = data.exp.abbr, Flag = baseFlag, Weights = finalWeights, Hash = trackHash }); + } else { + float defVal = data.exp.defaultValue; + + if (data.cMax > defVal + 0.1f) { + float[] rawWeights = new float[sampleCount]; + float[] finalWeights = new float[sampleCount]; + int trackHash = 17; + trackHash = unchecked(trackHash * 31 + HashString(data.exp.abbr)); + + for (int i = 0; i < sampleCount; i++) { + int tick = project.timeAxis.MsPosToTickPos(phonePosMs + (i * 5.0)) - activePart.position; + float val = data.c.Sample(tick); + float rw = val > defVal ? Math.Clamp(((val - defVal) / Math.Max(0.001f, data.exp.max - defVal)) * 100f, 0, 100) : 0; + rawWeights[i] = isVoiced[i] ? rw : (rw > 50f ? 100f : 0f); + } + + for (int i = 0; i < sampleCount; i++) { + if (i < 2 || i >= sampleCount - 2) finalWeights[i] = rawWeights[i]; + else finalWeights[i] = (rawWeights[i - 2] + rawWeights[i - 1] + rawWeights[i] + rawWeights[i + 1] + rawWeights[i + 2]) / 5f; + trackHash = unchecked(trackHash * 31 + (int)Math.Round(finalWeights[i])); + } + + string trkFlag = ""; + foreach(var d2 in dynamicExps) { + if (d2.color != null) continue; + string r2 = string.IsNullOrEmpty(d2.exp.flag) ? d2.exp.abbr : d2.exp.flag; + trkFlag += (d2.exp.abbr == data.exp.abbr) ? $"{r2}{(int)Math.Round((double)data.cMax)}" : $"{r2}{(int)Math.Round((double)d2.exp.defaultValue)}"; + } + baseHash = unchecked(baseHash * 31 + trackHash); + tracks.Add(new MorphTrack { TargetColor = null, Abbr = data.exp.abbr, Flag = trkFlag, Weights = finalWeights, Hash = trackHash }); + } + if (data.cMin < defVal - 0.1f) { + float[] rawWeights = new float[sampleCount]; + float[] finalWeights = new float[sampleCount]; + int trackHash = 17; + trackHash = unchecked(trackHash * 31 + HashString(data.exp.abbr)); + + for (int i = 0; i < sampleCount; i++) { + int tick = project.timeAxis.MsPosToTickPos(phonePosMs + (i * 5.0)) - activePart.position; + float val = data.c.Sample(tick); + float rw = val < defVal ? Math.Clamp(((defVal - val) / Math.Max(0.001f, defVal - data.exp.min)) * 100f, 0, 100) : 0; + rawWeights[i] = isVoiced[i] ? rw : (rw > 50f ? 100f : 0f); + } + + for (int i = 0; i < sampleCount; i++) { + if (i < 2 || i >= sampleCount - 2) finalWeights[i] = rawWeights[i]; + else finalWeights[i] = (rawWeights[i - 2] + rawWeights[i - 1] + rawWeights[i] + rawWeights[i + 1] + rawWeights[i + 2]) / 5f; + trackHash = unchecked(trackHash * 31 + (int)Math.Round(finalWeights[i])); + } + + string trkFlag = ""; + foreach(var d2 in dynamicExps) { + if (d2.color != null) continue; + string r2 = string.IsNullOrEmpty(d2.exp.flag) ? d2.exp.abbr : d2.exp.flag; + trkFlag += (d2.exp.abbr == data.exp.abbr) ? $"{r2}{(int)Math.Round((double)data.cMin)}" : $"{r2}{(int)Math.Round((double)d2.exp.defaultValue)}"; + } + baseHash = unchecked(baseHash * 31 + trackHash); + tracks.Add(new MorphTrack { TargetColor = null, Abbr = data.exp.abbr, Flag = trkFlag, Weights = finalWeights, Hash = trackHash }); + } + } + } + return tracks.Count > 0; + } + // Garbage collector for morphing cache, auto deletes them to prevent cache folder inflation + private void CleanUpMetaFiles(string filePath) { + if (string.IsNullOrEmpty(filePath)) return; + string ext = Path.GetExtension(filePath); + string noExt = filePath.Substring(0, filePath.Length - ext.Length); + string frqExt = ext.Replace('.', '_') + ".frq"; + + string[] pathsToDelete = new string[] { + noExt + frqExt, + filePath + ".llsm", + filePath + ".uspec", + filePath + ".dio", + filePath + ".star", + filePath + ".platinum", + filePath + ".frc", + filePath + ".pmk", + filePath + ".vs4ufrq", + noExt + ".rudb", + noExt + ".sc.npz", + noExt + ".sc", + noExt + ".hifi.npz" + }; + + foreach (string p in pathsToDelete) { + try { if (File.Exists(p)) File.Delete(p); } catch { } + } + } } -} +} \ No newline at end of file diff --git a/OpenUtau.Core/Classic/SpectralMorpher.cs b/OpenUtau.Core/Classic/SpectralMorpher.cs new file mode 100644 index 000000000..6c83a9d57 --- /dev/null +++ b/OpenUtau.Core/Classic/SpectralMorpher.cs @@ -0,0 +1,124 @@ +using System; +using System.Collections.Generic; +using System.Numerics; +using MathNet.Numerics.IntegralTransforms; +using OpenUtau.Core.Util; + +namespace OpenUtau.Core.Render { + public static class SpectralMorpher { + private const int N_FFT = 2048; + private const int HOP_LENGTH = 512; + + public static float[] MorphN(float[] baseAudio, List colorAudios, List colorCurves, int sampleRate = 44100) { + double[] window = new double[N_FFT]; + for (int i = 0; i < N_FFT; i++) window[i] = 0.5 * (1 - Math.Cos(2 * Math.PI * i / (N_FFT - 1))); + + Complex[][] stftBase = ComputeSTFT(baseAudio, window); + List stftColors = new List(); + foreach (var audio in colorAudios) stftColors.Add(ComputeSTFT(audio, window)); + + int numFrames = stftBase.Length; + Complex[][] stftMorphed = new Complex[numFrames][]; + int numColors = colorAudios.Count; + + for (int i = 0; i < numFrames; i++) { + stftMorphed[i] = new Complex[N_FFT]; + double timeMs = (i * HOP_LENGTH / (double)sampleRate) * 1000.0; + int curveIndex = (int)(timeMs / 5.0); + + double[] weights = new double[numColors]; + double sumColorWeights = 0; + for (int c = 0; c < numColors; c++) { + int idx = Math.Min(curveIndex, colorCurves[c].Length - 1); + weights[c] = Math.Max(0, colorCurves[c][idx]) / 100.0; + sumColorWeights += weights[c]; + } + if (sumColorWeights > 1.0) { + for (int c = 0; c < numColors; c++) weights[c] /= sumColorWeights; + sumColorWeights = 1.0; + } + double baseWeight = 1.0 - sumColorWeights; + + bool usePhaseLocked = Preferences.Default.PhaseLocked; + + if (usePhaseLocked) { + // Phase-Locked (Stable volume, better for smooth vowels) + for (int bin = 0; bin < N_FFT; bin++) { + double targetMag = baseWeight * stftBase[i][bin].Magnitude; + + for (int c = 0; c < numColors; c++) { + var frameColor = (i < stftColors[c].Length) ? stftColors[c][i] : stftBase[i]; + targetMag += weights[c] * frameColor[bin].Magnitude; + } + + stftMorphed[i][bin] = Complex.FromPolarCoordinates(targetMag, stftBase[i][bin].Phase); + } + } else { + // Complex Addition (Rich texture, better for growl/breath) + for (int bin = 0; bin < N_FFT; bin++) { + Complex targetComplex = stftBase[i][bin] * baseWeight; + + for (int c = 0; c < numColors; c++) { + var frameColor = (i < stftColors[c].Length) ? stftColors[c][i] : stftBase[i]; + targetComplex += frameColor[bin] * weights[c]; + } + + stftMorphed[i][bin] = targetComplex; + } + } + } + return ComputeISTFT(stftMorphed, window, baseAudio.Length); + } + + private static Complex[][] ComputeSTFT(float[] audio, double[] window) { + int numFrames = (audio.Length / HOP_LENGTH) + 3; + Complex[][] frames = new Complex[numFrames][]; + + for (int i = 0; i < numFrames; i++) { + frames[i] = new Complex[N_FFT]; + int offset = i * HOP_LENGTH - N_FFT / 2; + + for (int j = 0; j < N_FFT; j++) { + int idx = offset + j; + float sample = (idx >= 0 && idx < audio.Length) ? audio[idx] : 0f; + frames[i][j] = new Complex(sample * window[j], 0); + } + Fourier.Forward(frames[i], FourierOptions.Default); + } + return frames; + } + + private static float[] ComputeISTFT(Complex[][] stft, double[] window, int expectedLength) { + int numFrames = stft.Length; + int maxOutLength = numFrames * HOP_LENGTH + N_FFT; + float[] output = new float[maxOutLength]; + double[] windowSum = new double[maxOutLength]; + + for (int i = 0; i < numFrames; i++) { + Complex[] frame = new Complex[N_FFT]; + Array.Copy(stft[i], frame, N_FFT); + + Fourier.Inverse(frame, FourierOptions.Default); + + int offset = i * HOP_LENGTH - N_FFT / 2; + for (int j = 0; j < N_FFT; j++) { + int idx = offset + j; + if (idx >= 0 && idx < maxOutLength) { + output[idx] += (float)(frame[j].Real * window[j]); + windowSum[idx] += window[j] * window[j]; + } + } + } + + float[] finalOutput = new float[expectedLength]; + for (int i = 0; i < expectedLength; i++) { + double wSum = windowSum[i]; + float val = 0f; + if (wSum > 1e-8) val = (float)(output[i] / wSum); + if (float.IsNaN(val) || float.IsInfinity(val)) val = 0f; + finalOutput[i] = val; + } + return finalOutput; + } + } +} \ No newline at end of file diff --git a/OpenUtau.Core/OpenUtau.Core.csproj b/OpenUtau.Core/OpenUtau.Core.csproj index de3d0761f..9ae300369 100644 --- a/OpenUtau.Core/OpenUtau.Core.csproj +++ b/OpenUtau.Core/OpenUtau.Core.csproj @@ -14,6 +14,7 @@ + diff --git a/OpenUtau.Core/Render/RenderPhrase.cs b/OpenUtau.Core/Render/RenderPhrase.cs index 048fd64f6..feb772387 100644 --- a/OpenUtau.Core/Render/RenderPhrase.cs +++ b/OpenUtau.Core/Render/RenderPhrase.cs @@ -418,7 +418,7 @@ double Fade(double diff, int pit) { var curves = new List>(); foreach(var descriptor in project.expressions.Values) { - if(descriptor.type != UExpressionType.Curve) { + if(descriptor.type != UExpressionType.Curve && descriptor.type != UExpressionType.MorphingCurve) { continue; } var curve = part.curves.FirstOrDefault(c => c.abbr == descriptor.abbr); diff --git a/OpenUtau.Core/Ustx/UExpression.cs b/OpenUtau.Core/Ustx/UExpression.cs index 053fa2c5a..ef0cfb6a8 100644 --- a/OpenUtau.Core/Ustx/UExpression.cs +++ b/OpenUtau.Core/Ustx/UExpression.cs @@ -8,6 +8,7 @@ public enum UExpressionType : int { Numerical = 0, Options = 1, Curve = 2, + MorphingCurve = 3, } /// diff --git a/OpenUtau.Core/Util/Preferences.cs b/OpenUtau.Core/Util/Preferences.cs index fe4ff94b0..98a242c5e 100644 --- a/OpenUtau.Core/Util/Preferences.cs +++ b/OpenUtau.Core/Util/Preferences.cs @@ -250,6 +250,8 @@ public class SerializablePreferences { "; public string RecoveryPath = string.Empty; public bool DetachPianoRoll = false; + public bool PhraseLevelMorphing = false; + public bool PhaseLocked = true; // Legacy [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] diff --git a/OpenUtau/Controls/ExpressionCanvas.cs b/OpenUtau/Controls/ExpressionCanvas.cs index 901ff5def..e6ee132da 100644 --- a/OpenUtau/Controls/ExpressionCanvas.cs +++ b/OpenUtau/Controls/ExpressionCanvas.cs @@ -120,7 +120,7 @@ public override void Render(DrawingContext context) { double optionHeight = descriptor.type == UExpressionType.Options ? Bounds.Height / descriptor.options.Length : 0; - if (descriptor.type == UExpressionType.Curve) { + if (descriptor.type == UExpressionType.Curve || descriptor.type == UExpressionType.MorphingCurve) { var curve = Part.curves.FirstOrDefault(c => c.descriptor == descriptor); double defaultHeight = Math.Round(Bounds.Height - Bounds.Height * (descriptor.defaultValue - descriptor.min) / (descriptor.max - descriptor.min)); var lPen = ThemeManager.AccentPen1; diff --git a/OpenUtau/Controls/PianoRoll.axaml.cs b/OpenUtau/Controls/PianoRoll.axaml.cs index 8b4207ab2..751cc9618 100644 --- a/OpenUtau/Controls/PianoRoll.axaml.cs +++ b/OpenUtau/Controls/PianoRoll.axaml.cs @@ -981,7 +981,7 @@ public void ExpCanvasPointerPressed(object sender, PointerPressedEventArgs args) return; } if (point.Properties.IsLeftButtonPressed) { - if (descriptor.type == UExpressionType.Curve) { + if (descriptor.type == UExpressionType.Curve || descriptor.type == UExpressionType.MorphingCurve) { switch (ViewModel.CurveViewModel.CurveTool) { case CurveTools.CurveSelectTool: editState = new CurveSelectionState(control, ViewModel, this, descriptor); @@ -1003,7 +1003,7 @@ public void ExpCanvasPointerPressed(object sender, PointerPressedEventArgs args) } Cursor = null; } else if (point.Properties.IsRightButtonPressed) { - if (descriptor.type == UExpressionType.Curve && ViewModel.CurveViewModel.CurveTool == CurveTools.CurveSelectTool) { + if ((descriptor.type == UExpressionType.Curve || descriptor.type == UExpressionType.MorphingCurve) && ViewModel.CurveViewModel.CurveTool == CurveTools.CurveSelectTool) { ViewModel.CurveViewModel.ClearSelect(); } else { ViewModel.CurveViewModel.ClearSelect(); diff --git a/OpenUtau/Strings/Strings.axaml b/OpenUtau/Strings/Strings.axaml index 45eb4a17a..1c712235c 100644 --- a/OpenUtau/Strings/Strings.axaml +++ b/OpenUtau/Strings/Strings.axaml @@ -221,6 +221,7 @@ Do you want to continue by splitting at the nearest position after current playh Track Default Type Curve + Morphing Curve Numerical Options @@ -639,6 +640,8 @@ Warning: this option removes custom presets. Maximum Render Threads Wavtool UTAU + Phrase Level Morphing + Phase-locked Morphing Cache cleared. Clearing cache... diff --git a/OpenUtau/ViewModels/ExpressionsViewModel.cs b/OpenUtau/ViewModels/ExpressionsViewModel.cs index 51e29f948..fb1b7d296 100644 --- a/OpenUtau/ViewModels/ExpressionsViewModel.cs +++ b/OpenUtau/ViewModels/ExpressionsViewModel.cs @@ -65,8 +65,8 @@ public ExpressionBuilder(string name, string abbr, float min, float max, bool is this.WhenAnyValue(x => x.Abbr) .Select(abbr => !Core.Format.Ustx.required.Contains(abbr) || ExpressionsViewModel.isTrackOverride) .ToProperty(this, x => x.IsRemovable, out isRemovable); - this.WhenAnyValue(x => x.ExpressionType) - .Select(type => type == 0) // Numerical + this.WhenAnyValue(x => x.ExpressionType) + .Select(type => type == 0 || type == 3) // 0 = Numerical, 3 = MorphingCurve .ToProperty(this, x => x.IsNumerical, out isNumerical); this.WhenAnyValue(x => x.ExpressionType) .Select(type => type == 1) // Options @@ -83,7 +83,7 @@ public ExpressionBuilder(string name, string abbr, float min, float max, bool is if (string.IsNullOrWhiteSpace(Abbr)) { return new string[] { "Abbreviation must be set.", "" }; } - if (ExpressionType == 0) { // Numerical + if (ExpressionType == 0 || ExpressionType == 3) { // Numerical or MorphingCurve if (Abbr.Trim().Length < 1 || Abbr.Trim().Length > 4) { return new string[] { "Abbreviation must be between 1 and 4 characters long.", $": {Name}" }; } @@ -110,6 +110,10 @@ public UExpressionDescriptor Build() { return new UExpressionDescriptor(Name.Trim(), Abbr.Trim().ToLower(), Min, Max, DefaultValue) { type = UExpressionType.Curve, }; + case UExpressionType.MorphingCurve: + return new UExpressionDescriptor(Name.Trim(), Abbr.Trim().ToLower(), Min, Max, DefaultValue, Flag, CustomeDefaultValue, SkipOutputIfDefault) { + type = UExpressionType.MorphingCurve, + }; } throw new Exception("Unexpected expression type"); } diff --git a/OpenUtau/ViewModels/NotePropertiesViewModel.cs b/OpenUtau/ViewModels/NotePropertiesViewModel.cs index 465033a69..75c7d3eb0 100644 --- a/OpenUtau/ViewModels/NotePropertiesViewModel.cs +++ b/OpenUtau/ViewModels/NotePropertiesViewModel.cs @@ -184,7 +184,7 @@ public void LoadPart(UPart? part) { this.Part = part as UVoicePart; var track = DocManager.Inst.Project.tracks[part.trackNo]; foreach (var descriptor in track.GetSupportedExps(DocManager.Inst.Project)) { - if (descriptor.type != UExpressionType.Curve) { + if (descriptor.type != UExpressionType.Curve && descriptor.type != UExpressionType.MorphingCurve) { var viewModel = new NotePropertyExpViewModel(descriptor, this); if (descriptor.abbr == Ustx.CLR) { if (track.VoiceColorExp != null && track.VoiceColorExp.options.Length > 0) { diff --git a/OpenUtau/ViewModels/NotesViewModel.cs b/OpenUtau/ViewModels/NotesViewModel.cs index f81023e08..0602ef8b8 100644 --- a/OpenUtau/ViewModels/NotesViewModel.cs +++ b/OpenUtau/ViewModels/NotesViewModel.cs @@ -168,7 +168,7 @@ public NotesViewModel() { ExpTrackHeight = t.Item1.Height / descriptor.options.Length; ExpShadowOpacity = 0; } - ShowCurveToolbox = descriptor.type == UExpressionType.Curve; + ShowCurveToolbox = descriptor.type == UExpressionType.Curve || descriptor.type == UExpressionType.MorphingCurve; } else { ExpTrackHeight = 0; ExpShadowOpacity = 0.3; diff --git a/OpenUtau/ViewModels/PasteParamViewModel.cs b/OpenUtau/ViewModels/PasteParamViewModel.cs index 41a105c17..4d0ec7d25 100644 --- a/OpenUtau/ViewModels/PasteParamViewModel.cs +++ b/OpenUtau/ViewModels/PasteParamViewModel.cs @@ -9,7 +9,7 @@ public PasteParamViewModel() { Params.Add(new PasteParameter("pitch points", "")); Params.Add(new PasteParameter("vibrato", "")); foreach(var exp in DocManager.Inst.Project.expressions) { - if(exp.Value.type != Core.Ustx.UExpressionType.Curve) { + if(exp.Value.type != Core.Ustx.UExpressionType.Curve && exp.Value.type != Core.Ustx.UExpressionType.MorphingCurve) { Params.Add(new PasteParameter(exp.Value.name, exp.Key)); } } diff --git a/OpenUtau/ViewModels/PreferencesViewModel.cs b/OpenUtau/ViewModels/PreferencesViewModel.cs index 9db829b46..83139abaf 100644 --- a/OpenUtau/ViewModels/PreferencesViewModel.cs +++ b/OpenUtau/ViewModels/PreferencesViewModel.cs @@ -110,6 +110,8 @@ public int SafeMaxThreadCount { [Reactive] public int OtoEditor { get; set; } public string VLabelerPath => Preferences.Default.VLabelerPath; public string SetParamPath => Preferences.Default.SetParamPath; + [Reactive] public bool PhraseLevelMorphing { get; set; } + [Reactive] public bool PhaseLocked { get; set; } // Diffsinger public List DiffSingerStepsOptions { get; } = new List { 2, 5, 10, 20, 50, 100, 200, 500, 1000 }; @@ -121,7 +123,6 @@ public int SafeMaxThreadCount { [Reactive] public double DiffSingerDepth { get; set; } [Reactive] public bool DiffSingerTensorCache { get; set; } [Reactive] public bool DiffSingerLangCodeHide { get; set; } - // Advanced [Reactive] public bool RememberMid { get; set; } [Reactive] public bool RememberUst { get; set; } @@ -191,6 +192,8 @@ public PreferencesViewModel() { RememberUst = Preferences.Default.RememberUst; RememberVsqx = Preferences.Default.RememberVsqx; ClearCacheOnQuit = Preferences.Default.ClearCacheOnQuit; + PhraseLevelMorphing = Preferences.Default.PhraseLevelMorphing; + PhaseLocked = Preferences.Default.PhaseLocked; MessageBus.Current.Listen() .Subscribe(_ => this.RaisePropertyChanged(nameof(IsThemeEditorOpen))); @@ -403,6 +406,16 @@ public PreferencesViewModel() { Preferences.Default.SkipRenderingMutedTracks = skipRenderingMutedTracks; Preferences.Save(); }); + this.WhenAnyValue(vm => vm.PhraseLevelMorphing) + .Subscribe(index => { + Preferences.Default.PhraseLevelMorphing = index; + Preferences.Save(); + }); + this.WhenAnyValue(vm => vm.PhaseLocked) + .Subscribe(index => { + Preferences.Default.PhaseLocked = index; + Preferences.Save(); + }); } public void TestAudioOutputDevice() { diff --git a/OpenUtau/Views/ExpressionsDialog.axaml b/OpenUtau/Views/ExpressionsDialog.axaml index b2360fc13..36be79fbe 100644 --- a/OpenUtau/Views/ExpressionsDialog.axaml +++ b/OpenUtau/Views/ExpressionsDialog.axaml @@ -62,6 +62,7 @@ + diff --git a/OpenUtau/Views/NoteEditStates.cs b/OpenUtau/Views/NoteEditStates.cs index 0752dcdb6..6e4e9b01c 100644 --- a/OpenUtau/Views/NoteEditStates.cs +++ b/OpenUtau/Views/NoteEditStates.cs @@ -682,7 +682,7 @@ public override void Update(IPointer pointer, Point point) { if (descriptor == null) { return; } - if (descriptor.type != UExpressionType.Curve) { + if (descriptor.type != UExpressionType.Curve && descriptor.type != UExpressionType.MorphingCurve) { UpdatePhonemeExp(pointer, point); } else { UpdateCurveExp(pointer, point); @@ -808,7 +808,7 @@ public override void Update(IPointer pointer, Point point) { if (descriptor == null) { return; } - if (descriptor.type != UExpressionType.Curve) { + if (descriptor.type != UExpressionType.Curve && descriptor.type != UExpressionType.MorphingCurve) { ResetPhonemeExp(pointer, point); } else { ResetCurveExp(pointer, point); diff --git a/OpenUtau/Views/PreferencesDialog.axaml b/OpenUtau/Views/PreferencesDialog.axaml index e116bbe34..d30fde92b 100644 --- a/OpenUtau/Views/PreferencesDialog.axaml +++ b/OpenUtau/Views/PreferencesDialog.axaml @@ -299,6 +299,14 @@