Upgrade to Sentis 2.1

BoardGameAISample/Assets/Scripts/Othello.cs

@@ -13,8 +13,7 @@ public class Othello : MonoBehaviour
 {
     // Sentis
     public ModelAsset model;
-    IWorker m_Engine;
-    Ops m_Ops;
+    Worker m_Engine;
 
     // Board logic
     public AudioClip pieceDown, illegalMoveBuzzer;
@@ -57,9 +56,9 @@ enum CameraMove { TO_BOARD, TO_OPPONENTS, NONE };
     const int kRED = -1;  // Spirit
     int m_CurrentTurn = kBLACK; // keep track of who's turn it is to play
 
-    TensorFloat m_Data = TensorFloat.Zeros(new TensorShape(1, 1, kBoardDimension, kBoardDimension));
-    float[] m_LegalMoves = new float[kBoardDimension * kBoardDimension + 1];
-    TensorFloat m_MoveProbabilities = null;
+    Tensor<float> m_Data = new(new TensorShape(1, 1, kBoardDimension, kBoardDimension));
+    Tensor<float> m_LegalMoves = new(new TensorShape(kBoardDimension * kBoardDimension + 1));
+    Tensor<float> m_MoveProbabilities = null;
     GameObject[] m_Pieces = new GameObject[kBoardDimension * kBoardDimension];
 
     int m_PassesInARow = 0;
@@ -73,9 +72,29 @@ enum CameraMove { TO_BOARD, TO_OPPONENTS, NONE };
     void Start()
     {
         // Load in the neural network that will make the move predictions for the spirit + create inference engine
-        m_Engine = WorkerFactory.CreateWorker(BackendType.CPU, ModelLoader.Load(model));
-        // ops for tensor handling
-        m_Ops = new CPUOps();
+        var othelloModel = ModelLoader.Load(model);
+
+        var graph = new FunctionalGraph();
+        var inputs = graph.AddInputs(othelloModel);
+        var outputs = Functional.Forward(othelloModel, inputs);
+        var boardState = outputs[0];
+        var bestMove = outputs[1];
+
+        // Ensure legal moves are considered when computing best move.
+        var legal = graph.AddInput(DataType.Float, new TensorShape(kBoardDimension * kBoardDimension + 1));
+        // Convert outputs to probabilities
+        bestMove = Functional.Exp(bestMove * m_AIDifficultyTemperature);
+        // Mask out illegal moves
+        bestMove = Functional.Mul((0.0001f + bestMove), legal);
+        // Normalize probabilities so they sum to 1
+        var redSum = Functional.ReduceSum(bestMove, new int[] { 1 }, true);
+        bestMove /= redSum;
+
+        var bestMoveModel = graph.Compile(boardState, bestMove);
+        bestMoveModel.AddOutput("board_state", 0);
+        bestMoveModel.AddOutput("best_move", 1);
+
+        m_Engine = new Worker(bestMoveModel, BackendType.CPU);
 
         CreateBoard();
     }
@@ -121,7 +140,7 @@ void NextMove()
                 ComputerMove();
             }
         }
-        
+
     }
 
     Vector3 GetPiecePosition(int y, int x)
@@ -187,10 +206,10 @@ void ResetBoard()
         m_Data[(kBoardDimension / 2 - 1), (kBoardDimension / 2)] = kBLACK;
         m_Data[(kBoardDimension / 2), (kBoardDimension / 2 - 1)] = kBLACK;
         m_Data[(kBoardDimension / 2 - 1), (kBoardDimension / 2 - 1)] = kRED;
-        
+
         m_PassesInARow = 0;
         m_LastWinning = 0;
-        
+
         SetColors(kRED);
         m_CurrentTurn = kBLACK;
         SetSubtitle("Let's play. You begin.");
@@ -218,7 +237,7 @@ int SelectRandomMove()
 
     void FlipBoard()
     {
-        for (int i = 0; i < m_Data.shape.length; i++) 
+        for (int i = 0; i < m_Data.shape.length; i++)
             m_Data[i] *= -1;
     }
 
@@ -228,42 +247,25 @@ void ComputerMove()
         // The network is always form the point of view that the current player = 1 and opponent = -1
         FlipBoard();
 
-        m_Engine.Execute(m_Data);
-
-        m_Data.MakeReadable();
+        m_Engine.Schedule(m_Data, m_LegalMoves);
 
         // predict best move:
-        var bestMove = m_Engine.PeekOutput("best_move") as TensorFloat;
+        var bestMove = m_Engine.PeekOutput("best_move") as Tensor<float>;
         // estimate who is winning:
-        var boardState = m_Engine.PeekOutput("board_state") as TensorFloat;
+        var boardState = m_Engine.PeekOutput("board_state") as Tensor<float>;
 
-        boardState.MakeReadable();
         float boardValue = boardState[0, 0];
         bool blackIsWinning = -m_CurrentTurn * boardValue < 0;
 
         //convert the boardValue [-1,1] into a more human readable number:
         int percent = (int)(Mathf.Pow(Mathf.Abs(boardValue), 10f) * 50 + 50);
 
-        TensorFloat legal = new TensorFloat(new TensorShape(1, kBoardDimension * kBoardDimension + 1), m_LegalMoves);
-
         DisplayPhrases(blackIsWinning, percent);
 
-        // Convert outputs to probabilities:
-        bestMove = m_Ops.Exp(m_Ops.Mul(bestMove, m_AIDifficultyTemperature));
-        // Mask out illegal moves:
-        bestMove = m_Ops.Mul(m_Ops.Add(0.0001f, bestMove), legal);
-        // Normalize probabilities so they sum to 1
-        bestMove = m_Ops.Div(bestMove, m_Ops.ReduceSum(bestMove, new int[] { 1 }, true));
-
-        bestMove.MakeReadable();
-
-        m_MoveProbabilities = bestMove;
-        m_MoveProbabilities.TakeOwnership();
+        m_MoveProbabilities = bestMove.ReadbackAndClone();
 
         DisplayProbabilities();
 
-        legal?.Dispose();
-
         Invoke("MakeMove", m_PauseTime);
     }
 
@@ -351,7 +353,7 @@ void ClearProbabilityDisplay()
         }
     }
 
-    void MakeMove() 
+    void MakeMove()
     {
         ClearProbabilityDisplay();
         int moveIndex = SelectRandomMove();
@@ -413,7 +415,7 @@ int FlipColors(int y, int x, int turn, bool checkonly=false)
                 int enemyPieces = 0;
                 // check for a line of enemy pieces in direction (dx,dy):
                 while (Y >= 0 && X >= 0 && X < kBoardDimension && Y < kBoardDimension && m_Data[Y * kBoardDimension + X] == -turn)
-                {                  
+                {
                     X += dx; Y += dy;
                     enemyPieces++;
                 }
@@ -490,7 +492,7 @@ private void Update()
             float mouseY = Input.GetAxis("Mouse Y") * mouseSensititvy;
 
             cameraAngleLR = cameraAngleLR + mouseX;
-            
+
             cameraAngleUp = Mathf.Clamp(cameraAngleUp - mouseY, -45, 45);
             Camera.main.transform.localEulerAngles = new Vector3(cameraAngleUp, cameraAngleLR, 0);
         }
@@ -521,7 +523,7 @@ void MouseClicked()
         Ray ray = Camera.main.ScreenPointToRay(Input.mousePosition);
         if (!Physics.Raycast(ray, out RaycastHit hit, 1000))
             return;
-        
+
         GameObject go = hit.collider.gameObject;
 
         if (HasParent(go.transform, "Opponent Easy"))
@@ -536,7 +538,7 @@ void MouseClicked()
         {
             LevelOptionSelected(kRED, 2);
         }
-           
+
         int index = System.Array.IndexOf(m_Pieces, go);
         if (index < 0)
             return;
@@ -556,7 +558,7 @@ void MouseClicked()
         {
             GetComponent<AudioSource>().PlayOneShot(illegalMoveBuzzer);
             Debug.Log("Can't go there");
-        }  
+        }
     }
 
     private void OnApplicationQuit()
@@ -565,7 +567,6 @@ private void OnApplicationQuit()
         m_Engine?.Dispose();
         m_MoveProbabilities.Dispose();
         m_Data.Dispose();
-
-        m_Ops.Dispose();
+        m_LegalMoves.Dispose();
     }
 }

DepthEstimationSample/Assets/Scripts/InferenceWebcam.cs

@@ -1,53 +1,60 @@
 using System.Collections;
-using System.Collections.Generic;
 using UnityEngine;
 using Unity.Sentis;
 
 public class InferenceWebcam : MonoBehaviour
 {
     public ModelAsset estimationModel;
-    IWorker m_engineEstimation;
+    Worker m_engineEstimation;
     WebCamTexture webcamTexture;
-    TensorFloat inputTensor;
+    Tensor<float> inputTensor;
     RenderTexture outputTexture;
 
     public Material material;
     public Texture2D colorMap;
 
     int modelLayerCount = 0;
     public int framesToExectute = 2;
-    
+
     void Start()
     {
         Application.targetFrameRate = 60;
         var model = ModelLoader.Load(estimationModel);
-        var output = model.outputs[0];
-        model.layers.Add(new Unity.Sentis.Layers.ReduceMax("max0", new[] { output }, false));
-        model.layers.Add(new Unity.Sentis.Layers.ReduceMin("min0", new[] { output }, false));
-        model.layers.Add(new Unity.Sentis.Layers.Sub("maxO - minO", "max0", "min0"));
-        model.layers.Add(new Unity.Sentis.Layers.Sub("output - min0", output, "min0"));
-        model.layers.Add(new Unity.Sentis.Layers.Div("output2", "output - min0", "maxO - minO"));
+
+        // Post process
+        var graph = new FunctionalGraph();
+        var inputs = graph.AddInputs(model);
+        var outputs = Functional.Forward(model, inputs);
+        var output = outputs[0];
+
+        var max0 = Functional.ReduceMax(output, -1, false);
+        var min0 = Functional.ReduceMin(output, -1, false);
+        var subMax0Min0 = max0 - min0;
+        var subOutputMin0 = output - min0;
+        var output2 = subOutputMin0 / subMax0Min0;
+
+        model = graph.Compile(output2);
         modelLayerCount = model.layers.Count;
-        model.outputs = new List<string>() { "output2" };
-        m_engineEstimation = WorkerFactory.CreateWorker(BackendType.GPUCompute, model);
+
+        m_engineEstimation = new Worker(model, BackendType.GPUCompute);
 
         WebCamDevice[] devices = WebCamTexture.devices;
         webcamTexture = new WebCamTexture(1920, 1080);
         webcamTexture.deviceName = devices[0].name;
         webcamTexture.Play();
 
         outputTexture = new RenderTexture(256, 256, 0, RenderTextureFormat.ARGBFloat);
-        inputTensor = TensorFloat.Zeros(new TensorShape(1, 3, 256, 256));
+        inputTensor = new Tensor<float>(new TensorShape(1, 3, 256, 256));
     }
 
     bool executionStarted = false;
     IEnumerator executionSchedule;
-    private void Update()
+    void Update()
     {
         if (!executionStarted)
         {
             TextureConverter.ToTensor(webcamTexture, inputTensor, new TextureTransform());
-            executionSchedule = m_engineEstimation.StartManualSchedule(inputTensor);
+            executionSchedule = m_engineEstimation.ScheduleIterable(inputTensor);
             executionStarted = true;
         }
 
@@ -63,9 +70,9 @@ private void Update()
         if (hasMoreWork)
             return;
 
-        var output = m_engineEstimation.PeekOutput() as TensorFloat;
-        output = output.ShallowReshape(output.shape.Unsqueeze(0)) as TensorFloat;
-        TextureConverter.RenderToTexture(output as TensorFloat, outputTexture, new TextureTransform().SetCoordOrigin(CoordOrigin.BottomLeft));
+        var output = m_engineEstimation.PeekOutput() as Tensor<float>;
+        output.Reshape(output.shape.Unsqueeze(0));
+        TextureConverter.RenderToTexture(output, outputTexture, new TextureTransform().SetCoordOrigin(CoordOrigin.BottomLeft));
         executionStarted = false;
     }
 
@@ -78,7 +85,7 @@ void OnRenderObject()
         Graphics.Blit(null, material);
     }
 
-    private void OnDestroy()
+    void OnDestroy()
     {
         m_engineEstimation.Dispose();
         inputTensor.Dispose();

DigitRecognitionSample/Assets/Scripts/MNISTEngine.cs

@@ -6,10 +6,10 @@
 
 /*
  *  Neural net engine and handles the inference.
- *   - Shifts the image to the center for better inference. 
+ *   - Shifts the image to the center for better inference.
  *   (The model was trained on images centered in the texture this will give better results)
  *  - recentering of the image is also done using special operations on the GPU
- *   
+ *
  */
 
 // current bounds of the drawn image. It will help if we need to recenter the image later
@@ -26,7 +26,7 @@ public class MNISTEngine : MonoBehaviour
     public ModelAsset mnistONNX;
 
     // engine type
-    IWorker engine;
+    Worker engine;
 
     // This small model works just as fast on the CPU as well as the GPU:
     static Unity.Sentis.BackendType backendType = Unity.Sentis.BackendType.GPUCompute;
@@ -35,10 +35,7 @@ public class MNISTEngine : MonoBehaviour
     const int imageWidth = 28;
 
     // input tensor
-    TensorFloat inputTensor = null;
-
-    // op to manipulate Tensors 
-    Ops ops;
+    Tensor<float> inputTensor = null;
 
     Camera lookCamera;
 
@@ -47,11 +44,20 @@ void Start()
     {
         // load the neural network model from the asset:
         Model model = ModelLoader.Load(mnistONNX);
-        // create the neural network engine:
-        engine = WorkerFactory.CreateWorker(backendType, model);
 
-        // CreateOps allows direct operations on tensors.
-        ops = WorkerFactory.CreateOps(backendType, null);
+        var graph = new FunctionalGraph();
+        inputTensor = new Tensor<float>(new TensorShape(1, 1, imageWidth, imageWidth));
+        var input = graph.AddInput(DataType.Float, new TensorShape(1, 1, imageWidth, imageWidth));
+        var outputs = Functional.Forward(model, input);
+        var result = outputs[0];
+
+        // Convert the result to probabilities between 0..1 using the softmax function
+        var probabilities = Functional.Softmax(result);
+        var indexOfMaxProba = Functional.ArgMax(probabilities, -1, false);
+        model = graph.Compile(probabilities, indexOfMaxProba);
+
+        // create the neural network engine:
+        engine = new Worker(model, backendType);
 
         //The camera which we'll be using to calculate the rays on the image:
         lookCamera = Camera.main;
@@ -62,22 +68,15 @@ void Start()
     {
         inputTensor?.Dispose();
 
-        // Convert the texture into a tensor, it has width=W, height=W, and channels=1:    
+        // Convert the texture into a tensor, it has width=W, height=W, and channels=1:
         inputTensor = TextureConverter.ToTensor(drawableTexture, imageWidth, imageWidth, 1);
-        
+
         // run the neural network:
-        engine.Execute(inputTensor);
-
-        // We get a reference to the output of the neural network while keeping it on the GPU
-        TensorFloat result = engine.PeekOutput() as TensorFloat;
-
-        // convert the result to probabilities between 0..1 using the softmax function:
-        var probabilities = ops.Softmax(result);
-        var indexOfMaxProba = ops.ArgMax(probabilities, -1, false);
-
-        // We need to make the result from the GPU readable on the CPU
-        probabilities.MakeReadable();
-        indexOfMaxProba.MakeReadable();
+        engine.Schedule(inputTensor);
+
+        // We get a reference to the outputs of the neural network. Make the result from the GPU readable on the CPU
+        using var probabilities = (engine.PeekOutput(0) as Tensor<float>).ReadbackAndClone();
+        using var indexOfMaxProba = (engine.PeekOutput(1) as Tensor<int>).ReadbackAndClone();
 
         var predictedNumber = indexOfMaxProba[0];
         var probability = probabilities[predictedNumber];
@@ -120,13 +119,12 @@ void MouseIsDown()
             panel.ScreenGetMouse(hit);
         }
     }
-   
+
     // Clean up all our resources at the end of the session so we don't leave anything on the GPU or in memory:
     private void OnDestroy()
     {
         inputTensor?.Dispose();
         engine?.Dispose();
-        ops?.Dispose();
     }
 
 }