Merge branch 'main' of https://github.com/dfgHiatus/VRCFT-Babble

dfgHiatus · Sep 18, 2023 · fc24fa3 · fc24fa3
2 parents dabdf40 + d82b7c8
commit fc24fa3
Show file tree

Hide file tree

Showing 4 changed files with 166 additions and 42 deletions.
diff --git a/BabbleOSC.cs b/BabbleOSC.cs
@@ -1,12 +1,14 @@
-using System.Net;
+using System.Net;
 using System.Net.Sockets;
 using Microsoft.Extensions.Logging;
+using Newtonsoft.Json.Linq;
 using VRCFaceTracking.Core.OSC;
 
 namespace VRCFaceTracking.Babble;
 public partial class BabbleOSC
 {
     private Socket _receiver;
+    private bool _loop = true;
     private readonly Thread _thread;
     private readonly ILogger _logger;
     private readonly int _resolvedPort;
@@ -25,55 +27,142 @@ public BabbleOSC(ILogger iLogger, int? port = null)
 
         _receiver = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
         _resolvedPort = port ?? DEFAULT_PORT;
-        _receiver.Connect(new IPEndPoint(IPAddress.Parse("127.0.0.1"), _resolvedPort));
+        _receiver.Bind(new IPEndPoint(IPAddress.Parse("127.0.0.1"), _resolvedPort));
         _receiver.ReceiveTimeout = TIMEOUT_MS;
 
+        _loop = true;
         _thread = new Thread(new ThreadStart(ListenLoop));
         _thread.Start();
     }
 
+    struct Msg
+    {
+        public string address;
+        public float value;
+        public bool success;
+    }
+
+    // https://github.com/benaclejames/SimpleRustOSC/blob/master/src/lib.rs#L54
+    private Msg ParseOSC(byte[] buffer, int length)
+    {
+        Msg msg = new Msg();
+        msg.success = false;
+
+        if (length < 4)
+            return msg;
+
+        int bufferPosition = 0;
+        string address = ParseString(buffer, length, ref bufferPosition);
+        if (address == "")
+            return msg;
+
+        msg.address = address;
+
+        // checking for ',' char
+        if (buffer[bufferPosition] != 44)
+            return msg;
+        bufferPosition++; // skipping ',' character
+
+        char valueType = (char)buffer[bufferPosition]; // unused
+        bufferPosition++;
+
+        float value = ParesFloat(buffer, length, bufferPosition);
+
+        msg.value = value;
+        msg.success = true;
+
+        return msg;
+    }
+
+    private string ParseString(byte[] buffer, int length, ref int bufferPosition)
+    {
+        string address = "";
+
+        // first character must be '/'
+        if (buffer[0] != 47)
+            return address;
+
+        for (int i = 0; i < length; i++)
+        {
+            if (buffer[i] == 0)
+            {
+                bufferPosition = i + 1;
+
+                if (bufferPosition % 4 != 0)
+                {
+                    bufferPosition += 4 - (bufferPosition % 4);
+                }
+
+                break;
+            }
+
+            address += (char)buffer[i];
+        }
+
+        return address;
+    }
+
+    private float ParesFloat(byte[] buffer, int length, int bufferPosition)
+    {
+        var valueBuffer = new byte[length - bufferPosition];
+
+        int j = 0;
+        for (int i = bufferPosition; i < length; i++)
+        {
+            valueBuffer[j] = buffer[i];
+
+            j++;
+        }
+
+        float value = bytesToFLoat(valueBuffer);
+        return value;
+    }
+
+    private float bytesToFLoat(byte[] bytes)
+    {
+        if (BitConverter.IsLittleEndian)
+        {
+            Array.Reverse(bytes); // Convert big endian to little endian
+        }
+
+        float myFloat = BitConverter.ToSingle(bytes, 0);
+        return myFloat;
+    }
+
     private void ListenLoop()
     {
-        OscMessageMeta oscMeta = new OscMessageMeta();
         var buffer = new byte[4096];
 
-        while (!MainStandalone.MasterCancellationTokenSource.IsCancellationRequested)
+        while (_loop)
         {
             try
             {
-                if (_receiver.Connected)
+                if (_receiver.IsBound)
                 {
                     var length = _receiver.Receive(buffer);
-                    if (SROSCLib.parse_osc(buffer, length, ref oscMeta))
+
+                    Msg msg = ParseOSC(buffer, length);
+                    if (msg.success && BabbleExpressionMap.ContainsKey(msg.address))
                     {
-                        if (BabbleExpressionMap.ContainsKey(oscMeta.Address))
-                        {
-                            if (oscMeta.Type == OscValueType.Float) // Possibly redundant
-                            {
-                                BabbleExpressionMap[oscMeta.Address] = oscMeta.Value.FloatValues[0];
-                            }
-                        }
+                        BabbleExpressionMap[msg.address] = msg.value;
                     }
                 }
                 else
                 {
                     _receiver.Close();
                     _receiver.Dispose();
                     _receiver = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
-                    _receiver.Connect(new IPEndPoint(IPAddress.Parse("127.0.0.1"), _resolvedPort));
+                    _receiver.Bind(new IPEndPoint(IPAddress.Parse("127.0.0.1"), _resolvedPort));
                     _receiver.ReceiveTimeout = TIMEOUT_MS;
                 }
             }
-            catch (Exception e)
-            {
-                if (_receiver.Connected)
-                    _logger.LogError(e.Message);
-            }
+            catch (Exception) { }
         }
     }
 
     public void Teardown()
     {
+        _loop = false;
         _receiver.Close();
         _receiver.Dispose();
         _thread.Join();

diff --git a/BabbleVRC.cs b/BabbleVRC.cs
@@ -1,4 +1,4 @@
-using System.Reflection;
+using System.Reflection;
 using VRCFaceTracking.Core.Params.Expressions;
 
 namespace VRCFaceTracking.Babble;
@@ -19,23 +19,25 @@ public override (bool eyeSuccess, bool expressionSuccess) Initialize(bool eyeAva
         streams.Add(hmdStream);
         ModuleInformation = new ModuleMetadata()
         {
-            Name = "Project Babble Face Tracking\nInference Model v2.0.0"
+            Name = "Project Babble Face Tracking\nInference Model v2.0.0",
+            StaticImages = streams
         };
 
-        ModuleInformation.StaticImages = streams;
         return (false, true);
     }
 
     public override void Teardown() => babbleOSC.Teardown();
     public override void Update()
     {
+        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.TongueOut].Weight = babbleOSC.BabbleExpressionMap["/tongueOut"];
+
         UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.JawOpen].Weight = babbleOSC.BabbleExpressionMap["/jawOpen"];
-        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.JawLeft].Weight = babbleOSC.BabbleExpressionMap["/jawLeft"];
-        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.JawRight].Weight = babbleOSC.BabbleExpressionMap["/jawRight"];
-        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.JawForward].Weight = babbleOSC.BabbleExpressionMap["/jawOpen"];
+        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.JawLeft].Weight = -babbleOSC.BabbleExpressionMap["/jawLeft"];
+        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.JawRight].Weight = -babbleOSC.BabbleExpressionMap["/jawRight"];
+        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.JawForward].Weight = babbleOSC.BabbleExpressionMap["/jawForward"];
 
-        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.NoseSneerLeft].Weight = babbleOSC.BabbleExpressionMap["/noseSneerLeft"];
-        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.NoseSneerRight].Weight = babbleOSC.BabbleExpressionMap["/noseSneerRight"];
+        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.NoseSneerLeft].Weight = -babbleOSC.BabbleExpressionMap["/noseSneerLeft"];
+        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.NoseSneerRight].Weight = -babbleOSC.BabbleExpressionMap["/noseSneerRight"];
 
         UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.LipPuckerLowerLeft].Weight = babbleOSC.BabbleExpressionMap["/mouthPucker"];
         UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.LipPuckerLowerRight].Weight = babbleOSC.BabbleExpressionMap["/mouthPucker"];
@@ -47,23 +49,23 @@ public override void Update()
         UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.LipFunnelUpperLeft].Weight = babbleOSC.BabbleExpressionMap["/mouthFunnel"];
         UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.LipFunnelUpperRight].Weight = babbleOSC.BabbleExpressionMap["/mouthFunnel"];
 
-        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthUpperUpLeft].Weight = babbleOSC.BabbleExpressionMap["/mouthUpperUpLeft"];
-        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthUpperUpRight].Weight = babbleOSC.BabbleExpressionMap["/mouthUpperUpRight"];
-        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthLowerDownLeft].Weight = babbleOSC.BabbleExpressionMap["/mouthLowerDownLeft"];
-        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthLowerDownRight].Weight = babbleOSC.BabbleExpressionMap["/mouthLowerDownRight"];
+        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthUpperUpLeft].Weight = -babbleOSC.BabbleExpressionMap["/mouthUpperUpLeft"];
+        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthUpperUpRight].Weight = -babbleOSC.BabbleExpressionMap["/mouthUpperUpRight"];
+        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthLowerDownLeft].Weight = -babbleOSC.BabbleExpressionMap["/mouthLowerDownLeft"];
+        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthLowerDownRight].Weight = -babbleOSC.BabbleExpressionMap["/mouthLowerDownRight"];
 
-        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthPressLeft].Weight = babbleOSC.BabbleExpressionMap["/mouthPressLeft"];
-        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthPressRight].Weight = babbleOSC.BabbleExpressionMap["/mouthPressRight"];
+        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthPressLeft].Weight = -babbleOSC.BabbleExpressionMap["/mouthPressLeft"];
+        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthPressRight].Weight = -babbleOSC.BabbleExpressionMap["/mouthPressRight"];
 
-        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthStretchLeft].Weight = babbleOSC.BabbleExpressionMap["/mouthStretchLeft"];
-        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthStretchRight].Weight = babbleOSC.BabbleExpressionMap["/mouthStretchRight"];
-        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthDimpleLeft].Weight = babbleOSC.BabbleExpressionMap["/mouthDimpleLeft"];
-        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthDimpleRight].Weight = babbleOSC.BabbleExpressionMap["/mouthDimpleRight"];
+        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthStretchLeft].Weight = -babbleOSC.BabbleExpressionMap["/mouthStretchLeft"];
+        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthStretchRight].Weight = -babbleOSC.BabbleExpressionMap["/mouthStretchRight"];
+        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthDimpleLeft].Weight = -babbleOSC.BabbleExpressionMap["/mouthDimpleLeft"];
+        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthDimpleRight].Weight = -babbleOSC.BabbleExpressionMap["/mouthDimpleRight"];
 
-        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthCornerPullLeft].Weight = babbleOSC.BabbleExpressionMap["/mouthSmileLeft"];
-        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthCornerPullRight].Weight = babbleOSC.BabbleExpressionMap["/mouthSmileRight"];
-        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthFrownLeft].Weight = babbleOSC.BabbleExpressionMap["/mouthFrownLeft"];
-        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthFrownRight].Weight = babbleOSC.BabbleExpressionMap["/mouthFrownRight"];
+        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthCornerPullLeft].Weight = -babbleOSC.BabbleExpressionMap["/mouthSmileLeft"];
+        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthCornerPullRight].Weight = -babbleOSC.BabbleExpressionMap["/mouthSmileRight"];
+        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthFrownLeft].Weight = -babbleOSC.BabbleExpressionMap["/mouthFrownLeft"];
+        UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.MouthFrownRight].Weight = -babbleOSC.BabbleExpressionMap["/mouthFrownRight"];
         UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.CheekPuffLeft].Weight = babbleOSC.BabbleExpressionMap["/cheekPuff"];
         UnifiedTracking.Data.Shapes[(int)UnifiedExpressions.CheekPuffRight].Weight = babbleOSC.BabbleExpressionMap["/cheekPuff"];
     }

diff --git a/README.md b/README.md
@@ -1 +1,18 @@
-# VRCFT-Babble
+# VRCFT-Babble Module
+
+[Project Babble](https://github.com/SummerSigh/ProjectBabble) Module for [VRCFaceTracking](https://github.com/benaclejames/VRCFaceTracking). An open-source VR mouth-tracking solution
+
+![image](https://github.com/dfgHiatus/VRCFT-Babble/assets/51272212/90601f30-55a4-48af-a012-9cdb9bc47aa6)
+
+## Installation instructions
+
+1) Download and install [Project Babble](https://github.com/SummerSigh/ProjectBabble) and [VRCFaceTracking](https://github.com/benaclejames/VRCFaceTracking)
+1) Download a copy of the latest release [here](https://github.com/dfgHiatus/VRCFT-Babble/releases/latest)
+1) Drop the `VRCFaceTracking.Babble.dll` file in `AppData\Roaming\VRCFaceTracking\CustomLibs`. If this folder does not exist you can create it, VRCFaceTracking will create it on launch
+1) Launch Project Babble, and set your outgoing port to 8888. You can do this via the GUI, or by editing the Python script directly. Also, be sure to orient your camera properly so your expressions aren't flipped in-game
+1) Launch VRCFT (v5!)
+1) Enable OSC in VRChat
+1) Equip a face-tracking avatar (one with unified expressions)
+1) Profit!
+
+### Thanks to Summer#2406, Rames The Generic#3540, DenTechs#9511, PLYSHKA#6201, and benaclejames#6822 for helping me test, and encouraging me to do this!
diff --git a/manifest.json b/manifest.json
@@ -0,0 +1,16 @@
+{
+ "ModuleId": "360b014b-b57b-450f-8f12-9904618ff370",
+ "AuthorName": "dfgHiatus",
+ "DllFileName": "",
+ "Downloads": 0,
+ "DownloadUrl": "https://github.com/dfgHiatus/VRCFT-Babble/releases/download/v2.1.0a/VRCFaceTracking.Babble.dll",
+ "LastUpdated": "2023-04-20T01:08:01Z",
+ "ModuleDescription": "Project Babble face tracking for VRCFaceTracking v5.",
+ "ModuleName": "VRCFT-Babble",
+ "ModulePageUrl": "https://github.com/dfgHiatus/VRCFT-Babble", 
+ "Rating": 0, 
+ "Ratings": 0,
+ "RatingSum": 0,
+ "UsageInstructions": "For complete instructions, please refer to the project page.",
+ "Version": "1.0" 
+}