Merge pull request #34 from oxygen-dioxide/diffsinger

Diffsinger

OpenUtau.Core/DiffSinger/DiffSingerRenderer.cs

@@ -161,7 +161,7 @@ float[] InvokeDiffsinger(RenderPhrase phrase,int speedup) {
                 //get speaker curves
                 NDArray spkCurves = np.zeros<float>(totalFrames, speakers.Count);
                 foreach(var curve in phrase.curves) {
-                    if(IsVoiceColorCurve(curve.Item1,out int subBankId)) {
+                    if(IsVoiceColorCurve(curve.Item1,out int subBankId) && subBankId < singer.Subbanks.Count) {
                         var spkId = speakers.IndexOf(singer.Subbanks[subBankId].Suffix);
                         spkCurves[":", spkId] = DiffSingerUtils.SampleCurve(phrase, curve.Item2, 0, 
                             frameMs, totalFrames, headFrames, tailFrames, x => x * 0.01f)

OpenUtau.Core/DiffSinger/DiffSingerRhythmizerPhonemizer.cs

@@ -25,7 +25,6 @@ public class DsRhythmizerYaml {
     }
 
     class DsRhythmizer {
-        public string name;
         public string Location;
         public DsRhythmizerConfig config;
         public InferenceSession session;
@@ -47,13 +46,17 @@ public static Dictionary<string, string[]> LoadPhoneDict(string path, Encoding T
         }
 
         //通过名称获取音素时长模型
-        public DsRhythmizer(string name) {
-            this.name = name;
-            Location = Path.Combine(PathManager.Inst.DependencyPath, name);
+        public static DsRhythmizer FromName(string name) {
+            var path = Path.Combine(PathManager.Inst.DependencyPath, name);
+            return new DsRhythmizer(path);
+        }
+
+        public DsRhythmizer(string path) {
+            this.Location = path;
             config = Core.Yaml.DefaultDeserializer.Deserialize<DsRhythmizerConfig>(
                 File.ReadAllText(Path.Combine(Location, "vocoder.yaml"),
                     System.Text.Encoding.UTF8));
-            phoneDict = LoadPhoneDict(Path.Combine(Location,"dsdict.txt"), Encoding.UTF8);
+            phoneDict = LoadPhoneDict(Path.Combine(Location, "dsdict.txt"), Encoding.UTF8);
             //导入音素列表
             string phonemesPath = Path.Combine(Location, config.phonemes);
             phonemes = File.ReadLines(phonemesPath, Encoding.UTF8).ToList();
@@ -78,16 +81,20 @@ public override void SetSinger(USinger singer) {
             }
             //加载音素时长模型
             try {
-                string rhythmizerName;
-                string rhythmizerYamlPath = Path.Combine(singer.Location, "dsrhythmizer.yaml");
-                if (File.Exists(rhythmizerYamlPath)) {
-                    rhythmizerName = Core.Yaml.DefaultDeserializer.Deserialize<DsRhythmizerYaml>(
-                        File.ReadAllText(rhythmizerYamlPath, singer.TextFileEncoding)).rhythmizer;
-                } else {
-                    rhythmizerName = DsRhythmizer.DefaultRhythmizer;
-                }
-                if (rhythmizer == null || rhythmizer.name != rhythmizerName) {
-                    rhythmizer = new DsRhythmizer(rhythmizerName);
+                //if rhythmizer is packed within the voicebank
+                var packedRhythmizerPath = Path.Combine(singer.Location, "rhythmizer");
+                if(Directory.Exists(packedRhythmizerPath)) {
+                    rhythmizer = new DsRhythmizer(packedRhythmizerPath);
+                } else { 
+                    string rhythmizerName;
+                    string rhythmizerYamlPath = Path.Combine(singer.Location, "dsrhythmizer.yaml");
+                    if (File.Exists(rhythmizerYamlPath)) {
+                        rhythmizerName = Core.Yaml.DefaultDeserializer.Deserialize<DsRhythmizerYaml>(
+                            File.ReadAllText(rhythmizerYamlPath, singer.TextFileEncoding)).rhythmizer;
+                    } else {
+                        rhythmizerName = DsRhythmizer.DefaultRhythmizer;
+                    }
+                        rhythmizer = DsRhythmizer.FromName(rhythmizerName);
                 }
                 //导入拼音转音素字典，仅从时长模型包中导入字典
                 phoneDict = rhythmizer.phoneDict;

OpenUtau.Core/Util/TimeAxis.cs

@@ -104,8 +104,8 @@ public void BuildSegments(UProject project) {
                 tempoSegments[i].tickEnd = tempoSegments[i + 1].tickPos;
             }
             for (var i = 0; i < tempoSegments.Count; ++i) {
-                tempoSegments[i].msPerTick = 60.0 * 1000.0 * tempoSegments[i].beatPerBar / (tempoSegments[i].bpm * 4 * project.resolution);
-                tempoSegments[i].ticksPerMs = tempoSegments[i].bpm * 4 * project.resolution / (60.0 * 1000.0 * tempoSegments[i].beatPerBar);
+                tempoSegments[i].msPerTick = 60.0 * 1000.0 / (tempoSegments[i].bpm * project.resolution);
+                tempoSegments[i].ticksPerMs = tempoSegments[i].bpm * project.resolution / (60.0 * 1000.0);
                 if (i > 0) {
                     tempoSegments[i].msPos = tempoSegments[i - 1].msPos + tempoSegments[i - 1].Ticks * tempoSegments[i - 1].msPerTick;
                     tempoSegments[i - 1].msEnd = tempoSegments[i].msPos;

OpenUtau.Plugin.Builtin/EnunuOnnx/EnunuOnnxPhonemizer.cs

@@ -252,14 +252,30 @@ public string GetPhonemeType(string phoneme) {
         }
 
         string[] GetSymbols(Note note) {
-            if (string.IsNullOrEmpty(note.phoneticHint)) {
-                // User has not provided hint, query CMUdict.
-                return g2p.Query(note.lyric.ToLowerInvariant()) ?? new string[] {defaultPause};
-            }
-            // Split space-separated symbols into an array.
-            return note.phoneticHint.Split()
-                .Where(s => g2p.IsValidSymbol(s)) // skip the invalid symbols.
-                .ToArray();
+            //priority:
+            //1. phonetic hint
+            //2. query from g2p dictionary
+            //3. treat lyric as phonetic hint, including single phoneme
+            //4. default pause
+            if (!string.IsNullOrEmpty(note.phoneticHint)) {
+                // Split space-separated symbols into an array.
+                return note.phoneticHint.Split()
+                    .Where(s => g2p.IsValidSymbol(s)) // skip the invalid symbols.
+                    .ToArray();
+            }
+            // User has not provided hint, query g2p dictionary.
+            var g2presult = g2p.Query(note.lyric.ToLowerInvariant());
+            if(g2presult != null) {
+                return g2presult;
+            }
+            //not founded in g2p dictionary, treat lyric as phonetic hint
+            var lyricSplited = note.lyric.Split()
+                    .Where(s => g2p.IsValidSymbol(s)) // skip the invalid symbols.
+                    .ToArray();
+            if (lyricSplited.Length > 0) {
+                return lyricSplited;
+            }
+            return new string[] { defaultPause };
         }
 
         //make a HTS Note from given symbols and UNotes

OpenUtau.Test/Core/Util/TimeAxisTest.cs

@@ -26,15 +26,15 @@ public void ConvertMsTest() {
             Assert.Equal(2500, timeAxis.TickPosToMsPos(2400), 6);
             Assert.Equal(5000, timeAxis.TickPosToMsPos(4800), 6);
             Assert.Equal(9000, timeAxis.TickPosToMsPos(7200), 6);
-            Assert.Equal(20333.33333333, timeAxis.TickPosToMsPos(15960), 6);
-            Assert.Equal(35666.66666667, timeAxis.TickPosToMsPos(24000), 6);
+            Assert.Equal(21833.33333333, timeAxis.TickPosToMsPos(15960), 6);
+            Assert.Equal(37166.66666667, timeAxis.TickPosToMsPos(24000), 6);
 
             Assert.Equal(0, timeAxis.MsPosToTickPos(0));
             Assert.Equal(2400, timeAxis.MsPosToTickPos(2500));
             Assert.Equal(4800, timeAxis.MsPosToTickPos(5000));
             Assert.Equal(7200, timeAxis.MsPosToTickPos(9000));
-            Assert.Equal(15960, timeAxis.MsPosToTickPos(20333.33333333));
-            Assert.Equal(24000, timeAxis.MsPosToTickPos(35666.66666667));
+            Assert.Equal(15960, timeAxis.MsPosToTickPos(21833.33333333));
+            Assert.Equal(24000, timeAxis.MsPosToTickPos(37166.66666667));
         }
 
         [Fact]