Resampler Flag Curves and Classic Voice Color Curves via N-Way Spectral Morphing

OpenUtau.Core/Classic/ClassicRenderer.cs

@@ -18,8 +18,6 @@ public class ClassicRenderer : IRenderer {
             Ustx.DYN,
             Ustx.PITD,
             Ustx.CLR,
-            Ustx.CLRY,
-            Ustx.XSY,
             Ustx.ENG,
             Ustx.VEL,
             Ustx.VOL,
@@ -35,10 +33,13 @@ public class ClassicRenderer : IRenderer {
         public USingerType SingerType => USingerType.Classic;
 
         public bool SupportsRenderPitch => false;
+
         public bool SupportsExpression(UExpressionDescriptor descriptor) {
             return descriptor.isFlag
                 || !string.IsNullOrEmpty(descriptor.flag)
-                || supportedExp.Contains(descriptor.abbr);
+                || supportedExp.Contains(descriptor.abbr)
+                || descriptor.type == UExpressionType.MorphingCurve
+                || descriptor.abbr.StartsWith("cl", StringComparison.OrdinalIgnoreCase);
         }
 
         public RenderResult Layout(RenderPhrase phrase) {
@@ -58,19 +59,16 @@ public Task<RenderResult> Render(RenderPhrase phrase, Progress progress, int tra
         }
 
         public Task<RenderResult> RenderInternal(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender) {
-            var resamplerItems = new List<ResamplerItem>();
-            foreach (var phone in phrase.phones) {
-                resamplerItems.Add(new ResamplerItem(phrase, phone));
-            }
+            var resamplerItems = phrase.phones.Select(p => new ResamplerItem(phrase, p)).ToList();
             var task = Task.Run(() => {
-                Parallel.ForEach(source: resamplerItems, parallelOptions: new ParallelOptions() {
+                Parallel.ForEach(resamplerItems, new ParallelOptions {
                     MaxDegreeOfParallelism = Preferences.Default.NumRenderThreads
-                }, body: item => {
+                }, item => {
                     if (!cancellation.IsCancellationRequested && !File.Exists(item.outputFile)) {
                         if (!(item.resampler is WorldlineResampler)) {
                             VoicebankFiles.Inst.CopySourceTemp(item.inputFile, item.inputTemp);
                         }
-                        if(!item.phone.direct){
+                        if (!item.phone.direct) {
                             lock (Renderers.GetCacheLock(item.outputFile)) {
                                 item.resampler.DoResamplerReturnsFile(item, Log.Logger);
                             }
@@ -85,42 +83,73 @@ public Task<RenderResult> RenderInternal(RenderPhrase phrase, Progress progress,
                     }
                     progress.Complete(1, $"Track {trackNo + 1}: {item.resampler} \"{item.phone.phoneme}\"");
                 });
+
                 var result = Layout(phrase);
                 var wavtool = new SharpWavtool(true);
                 result.samples = wavtool.Concatenate(resamplerItems, string.Empty, cancellation);
                 if (result.samples != null) {
                     Renderers.ApplyDynamics(phrase, result);
-                    PlaybackManager.Inst.LiveWaveformCache[phrase.hash.ToString()] = (trackNo, phrase.positionMs - phrase.leadingMs, result.samples, DateTime.Now);
-                    Task.Factory.StartNew(() => {
-                        DocManager.Inst.ExecuteCmd(new WaveformReadyNotification());
-                    }, CancellationToken.None, TaskCreationOptions.None, DocManager.Inst.MainScheduler);
+                    if (phrase.renderSalt == 0) {
+                        PlaybackManager.Inst.LiveWaveformCache[phrase.hash.ToString()] = (trackNo, phrase.positionMs - phrase.leadingMs, result.samples, DateTime.Now);
+                        Task.Factory.StartNew(() => {
+                            DocManager.Inst.ExecuteCmd(new WaveformReadyNotification());
+                        }, CancellationToken.None, TaskCreationOptions.None, DocManager.Inst.MainScheduler);
+                    }
                 }
                 return result;
             });
             return task;
         }
 
         public Task<RenderResult> RenderExternal(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender) {
-            var resamplerItems = new List<ResamplerItem>();
-            foreach (var phone in phrase.phones) {
-                resamplerItems.Add(new ResamplerItem(phrase, phone));
-            }
+            var resamplerItems = phrase.phones.Select(p => new ResamplerItem(phrase, p)).ToList();
             var task = Task.Run(() => {
                 string progressInfo = $"Track {trackNo + 1} : {phrase.wavtool} \"{string.Join(" ", phrase.phones.Select(p => p.phoneme))}\"";
                 progress.Complete(0, progressInfo);
+
                 var wavPath = Path.Join(PathManager.Inst.CachePath, $"cat-{phrase.hash:x16}.wav");
                 phrase.AddCacheFile(wavPath);
                 var result = Layout(phrase);
-                if (File.Exists(wavPath)) {
-                    try {
-                        using (var waveStream = Wave.OpenFile(wavPath)) {
-                            result.samples = Wave.GetSamples(waveStream.ToSampleProvider().ToMono(1, 0));
+
+                lock (Renderers.GetCacheLock(wavPath)) {
+                    if (File.Exists(wavPath)) {
+                        try {
+                            using (var waveStream = Wave.OpenFile(wavPath)) {
+                                result.samples = Wave.GetSamples(waveStream.ToSampleProvider().ToMono(1, 0));
+                            }
+                        } catch (Exception e) {
+                            Log.Error(e, $"Failed to render: failed to open {wavPath}");
                         }
-                    } catch (Exception e) {
-                        Log.Error(e, $"Failed to render: failed to open {wavPath}");
                     }
                 }
+
                 if (result.samples == null) {
+                    Parallel.ForEach(resamplerItems, new ParallelOptions {
+                        MaxDegreeOfParallelism = Preferences.Default.NumRenderThreads
+                    }, item => {
+                        if (!cancellation.IsCancellationRequested && !File.Exists(item.outputFile)) {
+                            if (!(item.resampler is WorldlineResampler)) {
+                                VoicebankFiles.Inst.CopySourceTemp(item.inputFile, item.inputTemp);
+                            }
+                            if (!item.phone.direct) {
+                                lock (Renderers.GetCacheLock(item.outputFile)) {
+                                    item.resampler.DoResamplerReturnsFile(item, Log.Logger);
+                                }
+                                if (!File.Exists(item.outputFile)) {
+                                    DocManager.Inst.Project.timeAxis.TickPosToBarBeat(item.phrase.position + item.phone.position, out int bar, out int beat, out int tick);
+                                    throw new InvalidDataException($"{item.resampler} failed to resample \"{item.phone.phoneme}\" at {bar}:{beat}.{string.Format("{0:000}", tick)}");
+                                }
+                            }
+                            if (!(item.resampler is WorldlineResampler)) {
+                                VoicebankFiles.Inst.CopyBackMetaFiles(item.inputFile, item.inputTemp);
+                            }
+                        }
+                        progress.Complete(1, $"Track {trackNo + 1}: {item.resampler} \"{item.phone.phoneme}\"");
+                    });
+
+                    if (cancellation.IsCancellationRequested) return result;
+                    progress.Complete(0, $"Track {trackNo + 1}: {phrase.wavtool}");
+
                     foreach (var item in resamplerItems) {
                         VoicebankFiles.Inst.CopySourceTemp(item.inputFile, item.inputTemp);
                     }
@@ -130,13 +159,16 @@ public Task<RenderResult> RenderExternal(RenderPhrase phrase, Progress progress,
                         VoicebankFiles.Inst.CopyBackMetaFiles(item.inputFile, item.inputTemp);
                     }
                 }
+
                 progress.Complete(phrase.phones.Length, progressInfo);
                 if (result.samples != null) {
                     Renderers.ApplyDynamics(phrase, result);
-                    PlaybackManager.Inst.LiveWaveformCache[phrase.hash.ToString()] = (trackNo, phrase.positionMs - phrase.leadingMs, result.samples, DateTime.Now);
-                    Task.Factory.StartNew(() => {
-                        DocManager.Inst.ExecuteCmd(new WaveformReadyNotification());
-                    }, CancellationToken.None, TaskCreationOptions.None, DocManager.Inst.MainScheduler);
+                    if (phrase.renderSalt == 0) {
+                        PlaybackManager.Inst.LiveWaveformCache[phrase.hash.ToString()] = (trackNo, phrase.positionMs - phrase.leadingMs, result.samples, DateTime.Now);
+                        Task.Factory.StartNew(() => {
+                            DocManager.Inst.ExecuteCmd(new WaveformReadyNotification());
+                        }, CancellationToken.None, TaskCreationOptions.None, DocManager.Inst.MainScheduler);
+                    }
                 }
                 return result;
             });
@@ -148,13 +180,32 @@ public RenderPitchResult LoadRenderedPitch(RenderPhrase phrase) {
         }
 
         public UExpressionDescriptor[] GetSuggestedExpressions(USinger singer, URenderSettings renderSettings) {
-            var manifest= renderSettings.Resampler.Manifest;
-            if (manifest == null) {
-                return new UExpressionDescriptor[] { };
+            var expressions = new List<UExpressionDescriptor>();
+            var manifest = renderSettings.Resampler?.Manifest;
+            if (manifest != null && manifest.expressions != null) {
+                expressions.AddRange(manifest.expressions.Values);
+            }
+
+            if (singer != null && singer.Subbanks != null) {
+                var uniqueColors = singer.Subbanks.Select(s => s.Color).Where(c => !string.IsNullOrEmpty(c)).Distinct().ToList();
+                int colorIndex = 1;
+                foreach (var colorName in uniqueColors) {
+                    expressions.Add(new UExpressionDescriptor {
+                        name = $"voice color {colorIndex:D2} {colorName}",
+                        abbr = $"cl{colorIndex:D2}",
+                        type = UExpressionType.MorphingCurve,
+                        min = 0,
+                        max = 100,
+                        defaultValue = 0,
+                        isFlag = false,
+                        flag = ""
+                    });
+                    colorIndex++;
+                }
             }
-            return manifest.expressions.Values.ToArray();
+            return expressions.ToArray();
         }
 
         public override string ToString() => Renderers.CLASSIC;
     }
-}
+}

OpenUtau.Core/Classic/WorldlineRenderer.cs

@@ -4,7 +4,6 @@
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
-using System.Xml.Linq;
 using Microsoft.ML.OnnxRuntime;
 using Microsoft.ML.OnnxRuntime.Tensors;
 using NAudio.Wave;
@@ -34,8 +33,6 @@ public WorldlineRenderer(int version) {
             Ustx.DYN,
             Ustx.PITD,
             Ustx.CLR,
-            Ustx.CLRY,
-            Ustx.XSY,
             Ustx.SHFT,
             Ustx.VEL,
             Ustx.VOL,
@@ -54,7 +51,11 @@ public WorldlineRenderer(int version) {
         public bool SupportsRenderPitch => false;
 
         public bool SupportsExpression(UExpressionDescriptor descriptor) {
-            return supportedExp.Contains(descriptor.abbr);
+            return descriptor.isFlag
+                || !string.IsNullOrEmpty(descriptor.flag)
+                || supportedExp.Contains(descriptor.abbr)
+                || descriptor.type == UExpressionType.MorphingCurve
+                || descriptor.abbr.StartsWith("cl", StringComparison.OrdinalIgnoreCase);
         }
 
         public RenderResult Layout(RenderPhrase phrase) {
@@ -76,11 +77,17 @@ public Task<RenderResult> Render(RenderPhrase phrase, Progress progress, int tra
                 phrase.AddCacheFile(wavPath);
                 string progressInfo = $"Track {trackNo + 1}: {this} {string.Join(" ", phrase.phones.Select(p => p.phoneme))}";
                 progress.Complete(0, progressInfo);
-                if (File.Exists(wavPath)) {
-                    using (var waveStream = Wave.OpenFile(wavPath)) {
-                        result.samples = Wave.GetSamples(waveStream.ToSampleProvider().ToMono(1, 0));
+
+                lock (Renderers.GetCacheLock(wavPath)) {
+                    if (File.Exists(wavPath)) {
+                        try {
+                            using (var waveStream = Wave.OpenFile(wavPath)) {
+                                result.samples = Wave.GetSamples(waveStream.ToSampleProvider().ToMono(1, 0));
+                            }
+                        } catch { }
                     }
                 }
+
                 if (result.samples == null) {
                     var phraseSynth = new Worldline.PhraseSynthV2(44100, version == 1 ? 441 : 512, 2048);
                     double posOffsetMs = phrase.positionMs - phrase.leadingMs;
@@ -96,19 +103,19 @@ public Task<RenderResult> Render(RenderPhrase phrase, Progress progress, int tra
                         try {
                             phraseSynth.AddRequest(item, posMs, skipMs, lengthMs, fadeInMs, fadeOutMs);
                         } catch (SynthRequestError e) {
-                            if (e is CutOffExceedDurationError cee) {
+                            if (e is CutOffExceedDurationError) {
                                 throw new MessageCustomizableException(
                                     $"Failed to render\n Oto error: cutoff exceeds audio duration \n{item.phone.phoneme}",
                                     $"<translate:errors.failed.synth.cutoffexceedduration>\n{item.phone.phoneme}",
                                     e);
                             }
-                            if (e is CutOffBeforeOffsetError cbe) {
+                            if (e is CutOffBeforeOffsetError) {
                                 throw new MessageCustomizableException(
                                     $"Failed to render\n Oto error: cutoff before offset \n{item.phone.phoneme}",
                                     $"<translate:errors.failed.synth.cutoffbeforeoffset>\n{item.phone.phoneme}",
                                     e);
                             }
-                            throw e;
+                            throw;
                         }
                     }
                     int frames = (int)Math.Ceiling(result.estimatedLengthMs / frameMs);
@@ -182,9 +189,11 @@ public Task<RenderResult> Render(RenderPhrase phrase, Progress progress, int tra
                         var samplesCopy = (float[])result.samples.Clone();
                         Task.Run(() => {
                             try {
-                                var source = new WaveSource(0, 0, 0, 1);
-                                source.SetSamples(samplesCopy);
-                                WaveFileWriter.CreateWaveFile16(wavPath, new ExportAdapter(source).ToMono(1, 0));
+                                lock (Renderers.GetCacheLock(wavPath)) {
+                                    var source = new WaveSource(0, 0, 0, 1);
+                                    source.SetSamples(samplesCopy);
+                                    WaveFileWriter.CreateWaveFile16(wavPath, new ExportAdapter(source).ToMono(1, 0));
+                                }
                             } catch (Exception e) {
                                 Serilog.Log.Error(e, $"Failed to write cache file: {wavPath}");
                             }
@@ -256,10 +265,27 @@ public RenderPitchResult LoadRenderedPitch(RenderPhrase phrase) {
         }
 
         public UExpressionDescriptor[] GetSuggestedExpressions(USinger singer, URenderSettings renderSettings) {
-            return new UExpressionDescriptor[] { };
+            var expressions = new List<UExpressionDescriptor>();
+            if (singer != null && singer.Subbanks != null) {
+                var uniqueColors = singer.Subbanks.Select(s => s.Color).Where(c => !string.IsNullOrEmpty(c)).Distinct().ToList();
+                int colorIndex = 1;
+                foreach (var colorName in uniqueColors) {
+                    expressions.Add(new UExpressionDescriptor {
+                        name = $"voice color {colorIndex:D2} {colorName}",
+                        abbr = $"cl{colorIndex:D2}",
+                        type = UExpressionType.MorphingCurve,
+                        min = 0,
+                        max = 100,
+                        defaultValue = 0,
+                        isFlag = false,
+                        flag = ""
+                    });
+                    colorIndex++;
+                }
+            }
+            return expressions.ToArray();
         }
 
         public override string ToString() => version == 1 ? Renderers.WORLDLINE_R : Renderers.WORLDLINE_R2;
     }
-}
-
+}

OpenUtau.Core/Format/USTx.cs

@@ -10,7 +10,7 @@
 
 namespace OpenUtau.Core.Format {
     public class Ustx {
-        public static readonly Version kUstxVersion = new Version(0, 9);
+        public static readonly Version kUstxVersion = new Version(0, 10);
 
         public const string DYN = "dyn";
         public const string PITD = "pitd";
@@ -34,8 +34,6 @@ public class Ustx {
         public const string SHFC = "shfc";
         public const string TENC = "tenc";
         public const string VOIC = "voic";
-        public const string CLRY = "clry";
-        public const string XSY = "xsy";
 
         public static readonly string[] required = { DYN, PITD, CLR, ENG, VEL, VOL, ATK, DEC };
 
@@ -62,9 +60,6 @@ public static void AddDefaultExpressions(UProject project) {
             project.RegisterExpression(new UExpressionDescriptor("tone shift (curve)", SHFC, -1200, 1200, 0) { type = UExpressionType.Curve });
             project.RegisterExpression(new UExpressionDescriptor("tension (curve)", TENC, -100, 100, 0) { type = UExpressionType.Curve });
             project.RegisterExpression(new UExpressionDescriptor("voicing (curve)", VOIC, 0, 100, 100) { type = UExpressionType.Curve });
-            project.RegisterExpression(new UExpressionDescriptor("voice color y", CLRY, false, new string[0]));
-            project.RegisterExpression(new UExpressionDescriptor("cross synthesis (curve)", XSY, 0, 100, 0) { type = UExpressionType.Curve });
-
             string message = string.Empty;
             if (ValidateExpression(project, "g", GEN)) {
                 message += $"\ng flag -> gender";

OpenUtau.Core/OpenUtau.Core.csproj

@@ -15,6 +15,7 @@
     <PackageReference Include="csharp-pinyin" Version="1.0.0" />
     <PackageReference Include="Ignore" Version="0.1.50" />
     <PackageReference Include="K4os.Hash.xxHash" Version="1.0.8" />
+    <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
     <PackageReference Include="Melanchall.DryWetMidi" Version="7.2.0" />
     <PackageReference Include="NAudio.Core" Version="2.2.1" />
     <PackageReference Include="NAudio.Vorbis" Version="1.5.0" />