From bf470be048068b498c3cafd03f84c45c039faa20 Mon Sep 17 00:00:00 2001 From: Deepak Kumar Date: Tue, 16 Apr 2019 11:25:01 +0930 Subject: [PATCH] TF.NET backend in progress --- .../SiaNet.Backend.TensorFlow.csproj | 2 +- .../SiaNetBackend.cs | 100 +++++++++++------- .../SiaNet.Backend.Torch.csproj | 4 + Examples/BackendTester/Program.cs | 38 ++++++- .../Program.cs | 9 +- .../BostonHousingRegressionExample/Program.cs | 2 +- Examples/GettingStarted/Program.cs | 2 +- SiaNet/Data/DataFrame.cs | 40 ++++++- SiaNet/Data/DataFrame2D.cs | 2 +- SiaNet/Data/DataFrameIter.cs | 2 +- SiaNet/Data/ImageFrame.cs | 2 +- SiaNet/SiaNet.csproj | 4 +- SiaNet/TrainPredict.cs | 16 +-- 13 files changed, 162 insertions(+), 61 deletions(-) diff --git a/Backends/SiaNet.Backend.TensorFlow/SiaNet.Backend.TensorFlow.csproj b/Backends/SiaNet.Backend.TensorFlow/SiaNet.Backend.TensorFlow.csproj index 16c67fa..c8f5469 100644 --- a/Backends/SiaNet.Backend.TensorFlow/SiaNet.Backend.TensorFlow.csproj +++ b/Backends/SiaNet.Backend.TensorFlow/SiaNet.Backend.TensorFlow.csproj @@ -23,7 +23,7 @@ - + diff --git a/Backends/SiaNet.Backend.TensorFlow/SiaNetBackend.cs b/Backends/SiaNet.Backend.TensorFlow/SiaNetBackend.cs index 8659558..e81721e 100644 --- a/Backends/SiaNet.Backend.TensorFlow/SiaNetBackend.cs +++ b/Backends/SiaNet.Backend.TensorFlow/SiaNetBackend.cs @@ -123,7 +123,7 @@ public Engine.DataType GetDataType(SiaTensor x) public SiaTensor CreateVariable(float[] data, long[] shape, string name = "") { - return null; + return Out(tf.Variable(data).value()); } public SiaTensor Reshape(SiaTensor x, params long[] shape) @@ -348,7 +348,7 @@ public SiaTensor Round(SiaTensor x) public SiaTensor Sin(SiaTensor x) { - throw new NotImplementedException(); + return Out(tf.sin(In(x))); } public SiaTensor Cos(SiaTensor x) @@ -358,37 +358,37 @@ public SiaTensor Cos(SiaTensor x) public SiaTensor Tan(SiaTensor x) { - throw new NotImplementedException(); + return Out(tf.tan(In(x))); } public SiaTensor Asin(SiaTensor x) { - throw new NotImplementedException(); + return Out(tf.asin(In(x))); } public SiaTensor Acos(SiaTensor x) { - throw new NotImplementedException(); + return Out(tf.acos(In(x))); } public SiaTensor Atan(SiaTensor x) { - throw new NotImplementedException(); + return Out(tf.atan(In(x))); } public SiaTensor Sinh(SiaTensor x) { - throw new NotImplementedException(); + return Out(tf.sinh(In(x))); } public SiaTensor Cosh(SiaTensor x) { - throw new NotImplementedException(); + return Out(tf.cosh(In(x))); } public SiaTensor Tanh(SiaTensor x) { - throw new NotImplementedException(); + return Out(tf.tanh(In(x))); } public SiaTensor Sigmoid(SiaTensor x) @@ -398,47 +398,60 @@ public SiaTensor Sigmoid(SiaTensor x) public SiaTensor Pow(SiaTensor x, float value) { - throw new NotImplementedException(); + return Out(tf.pow(In(x), In(value, x.Shape))); } public SiaTensor Square(SiaTensor x) { - throw new NotImplementedException(); + return Out(tf.pow(In(x), 2)); } public SiaTensor Clip(SiaTensor x, float min, float max) { - throw new NotImplementedException(); + return Out(tf._clip_by_value(In(x), In(min, x.Shape), In(max, x.Shape))); } public float Sum(SiaTensor x) { - throw new NotImplementedException(); + return tf.reduce_sum(In(x)); } public SiaTensor Sum(SiaTensor x, int dim) { - throw new NotImplementedException(); + dim = dim < 0 ? x.DimCount + dim : dim; + return Out(tf.sum(In(x), dim, true)); } - public SiaTensor Sum(SiaTensor x, params int[] dim) + public SiaTensor Sum(SiaTensor x, params int[] dims) { - throw new NotImplementedException(); + foreach (var item in dims) + { + int dim = item < 0 ? x.DimCount + item : item; + x = Sum(x, item); + } + + return x; } public float Max(SiaTensor x) { - throw new NotImplementedException(); + return math_ops.reduce_max(In(x)); } public SiaTensor Max(SiaTensor x, int dim) { - throw new NotImplementedException(); + dim = dim < 0 ? x.DimCount + dim : dim; + return Out(math_ops.reduce_max(In(x), new int[] { dim }, true)); } - public SiaTensor Max(SiaTensor x, params int[] dim) + public SiaTensor Max(SiaTensor x, params int[] dims) { - throw new NotImplementedException(); + for (int i = 0; i < dims.Length; i++) + { + dims[i] = dims[i] < 0 ? x.DimCount + dims[i] : dims[i]; + } + + return Out(math_ops.reduce_max(In(x), dims, true)); } public float Min(SiaTensor x) @@ -458,62 +471,70 @@ public SiaTensor Min(SiaTensor x, params int[] dim) public float Mean(SiaTensor x) { - throw new NotImplementedException(); + return math_ops.reduce_mean(In(x)); } public SiaTensor Mean(SiaTensor x, int dim) { - throw new NotImplementedException(); + dim = dim < 0 ? x.DimCount + dim : dim; + return Out(math_ops.reduce_mean(In(x), new int[] { dim }, true)); } - public SiaTensor Mean(SiaTensor x, params int[] dim) + public SiaTensor Mean(SiaTensor x, params int[] dims) { - throw new NotImplementedException(); + for (int i = 0; i < dims.Length; i++) + { + dims[i] = dims[i] < 0 ? x.DimCount + dims[i] : dims[i]; + } + + return Out(math_ops.reduce_mean(In(x), dims, true)); } public SiaTensor Argmax(SiaTensor x, int dim = 0) { - throw new NotImplementedException(); + dim = dim < 0 ? x.DimCount + dim : dim; + return Out(tf.arg_max(In(x), dim)); } public SiaTensor Argmin(SiaTensor x, int dim = 0) { - throw new NotImplementedException(); + dim = dim < 0 ? x.DimCount + dim : dim; + return Out(tf.arg_min(In(x), dim)); } public SiaTensor Maximum(SiaTensor a, SiaTensor b) { - throw new NotImplementedException(); + return Out(tf.maximum(In(a), In(b))); } public SiaTensor Maximum(SiaTensor a, float b) { - throw new NotImplementedException(); + return Out(tf.maximum(In(a), In(b, a.Shape))); } public SiaTensor Minimum(SiaTensor a, SiaTensor b) { - throw new NotImplementedException(); + return Out(tf.minimum(In(a), In(b))); } public SiaTensor Minimum(SiaTensor a, float b) { - throw new NotImplementedException(); + return Out(tf.minimum(In(a), In(b, a.Shape))); } public SiaTensor Transpose(SiaTensor x) { - throw new NotImplementedException(); + return Out(tf.transpose(In(x), new int[] { 1, 0 })); } public SiaTensor Transpose(SiaTensor x, params int[] dims) { - throw new NotImplementedException(); + return Out(tf.transpose(In(x), dims)); } public SiaTensor Dot(SiaTensor a, SiaTensor b) { - throw new NotImplementedException(); + return Out(tf.matmul(In(a), In(b))); } public SiaTensor Diag(SiaTensor x) @@ -523,17 +544,18 @@ public SiaTensor Diag(SiaTensor x) public SiaTensor Softmax(SiaTensor x, int axis = -1) { - throw new NotImplementedException(); + return Out(tf.nn.softmax(In(x), axis)); } public SiaTensor Softplus(SiaTensor x, int axis = -1) { - throw new NotImplementedException(); + return Log((Exp(x) + 1)); } public SiaTensor L2Normalize(SiaTensor x, int axis = -1) { - throw new NotImplementedException(); + var y = Max(Sum(Square(x), axis), axis); + return x / Sqrt(y); } public SiaTensor Im2Col(SiaTensor x, Tuple kernalSize, int padding = 1, int stride = 1) @@ -558,17 +580,17 @@ public SiaTensor SliceCols(SiaTensor x, long start, long end) public Array GetArray(SiaTensor x) { - throw new NotImplementedException(); + return In(x).Data(); } public void Dispose(SiaTensor x) { - throw new NotImplementedException(); + In(x).Dispose(); } public ActivationFunc GetActFunc() { - throw new NotImplementedException(); + return new SiaNetActivations(this); } } } diff --git a/Backends/SiaNet.Backend.Torch/SiaNet.Backend.Torch.csproj b/Backends/SiaNet.Backend.Torch/SiaNet.Backend.Torch.csproj index bf66ced..d7b9677 100644 --- a/Backends/SiaNet.Backend.Torch/SiaNet.Backend.Torch.csproj +++ b/Backends/SiaNet.Backend.Torch/SiaNet.Backend.Torch.csproj @@ -8,6 +8,10 @@ true + + true + + diff --git a/Examples/BackendTester/Program.cs b/Examples/BackendTester/Program.cs index 1179f40..7ce961c 100644 --- a/Examples/BackendTester/Program.cs +++ b/Examples/BackendTester/Program.cs @@ -1,8 +1,10 @@ -using NumSharp.Core; +using CNTK; +using NumSharp; using SiaNet; using SiaNet.Engine; using SiaNet.Initializers; using System; +using System.Collections.Generic; using System.Linq; namespace BackendTester @@ -11,6 +13,15 @@ class Program { static void Main(string[] args) { + int[] shape = new int[] { 6, 3 }; + float[] data = new float[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + + NDArrayView array = new NDArrayView(shape, data, DeviceDescriptor.CPUDevice); + Variable variable = new Variable(shape, VariableKind.Parameter, CNTK.DataType.Float, array, false, new AxisVector(), false, "", ""); + var slicedData = CNTKLib.Slice(variable, AxisVector.Repeat(new Axis(0), 1), IntVector.Repeat(1, 1), IntVector.Repeat(3, 1)); + var resultArray = GetArray(slicedData); + + Global.UseEngine(SiaNet.Backend.CNTKLib.SiaNetBackend.Instance, DeviceType.CPU); var K = Global.CurrentBackend; @@ -25,5 +36,30 @@ static void Main(string[] args) Console.ReadLine(); } + + public static Array GetArray(Variable xvar) + { + Value v = null; + if (xvar.IsOutput) + { + var f = xvar.ToFunction(); + + var plist = f.Parameters(); + Dictionary inputs = new Dictionary(); + Dictionary outputs = new Dictionary() + { + { f, null} + }; + + f.Evaluate(inputs, outputs, DeviceDescriptor.CPUDevice); + v = outputs.FirstOrDefault().Value; + } + else + { + v = new Value(xvar.GetValue()); + } + + return v.GetDenseData(xvar)[0].ToArray(); + } } } diff --git a/Examples/BasicClassificationWithTitanicDataset/Program.cs b/Examples/BasicClassificationWithTitanicDataset/Program.cs index ab2b3e3..dd9a882 100644 --- a/Examples/BasicClassificationWithTitanicDataset/Program.cs +++ b/Examples/BasicClassificationWithTitanicDataset/Program.cs @@ -16,11 +16,11 @@ class Program static void Main(string[] args) { //Setup Engine - Global.UseEngine(SiaNet.Backend.MxNetLib.SiaNetBackend.Instance, DeviceType.Default); + Global.UseEngine(SiaNet.Backend.ArrayFire.SiaNetBackend.Instance, DeviceType.Default); var train = LoadTrain(); //Load train data var test = LoadTest(); //Load test data - + var model = new Sequential(); model.EpochEnd += Model_EpochEnd; model.Add(new Dense(128, ActType.ReLU)); @@ -31,10 +31,9 @@ static void Main(string[] args) model.Compile(OptimizerType.Adam, LossType.BinaryCrossEntropy, MetricType.BinaryAccurary); // Perform training with train and val dataset - model.Train(train, epochs: 100, batchSize: 32); + model.Train(train, epochs: 100, batchSize: 200); - //var prediction = model.Predict(test); - //TOps.Round(prediction).Print(); + var prediction = model.Predict(test); } private static void Model_EpochEnd(object sender, EpochEndEventArgs e) diff --git a/Examples/BostonHousingRegressionExample/Program.cs b/Examples/BostonHousingRegressionExample/Program.cs index 3714b40..fcb2049 100644 --- a/Examples/BostonHousingRegressionExample/Program.cs +++ b/Examples/BostonHousingRegressionExample/Program.cs @@ -13,7 +13,7 @@ class Program static void Main(string[] args) { //Setup Engine - Global.UseEngine(SiaNet.Backend.MxNetLib.SiaNetBackend.Instance, DeviceType.CPU); + Global.UseEngine(SiaNet.Backend.ArrayFire.SiaNetBackend.Instance, DeviceType.CPU); //Load Train and Test CSV data var ds = LoadTrain("./train.csv"); diff --git a/Examples/GettingStarted/Program.cs b/Examples/GettingStarted/Program.cs index 498aa79..509e574 100644 --- a/Examples/GettingStarted/Program.cs +++ b/Examples/GettingStarted/Program.cs @@ -38,7 +38,7 @@ static void Main(string[] args) predX.Load(0, 0, 0, 1); //Result should be 0 and 1 var rawPred = model.Predict(predX); - Global.CurrentBackend.Round(rawPred).Print(); + Console.ReadLine(); } diff --git a/SiaNet/Data/DataFrame.cs b/SiaNet/Data/DataFrame.cs index 1d55abf..e123c58 100644 --- a/SiaNet/Data/DataFrame.cs +++ b/SiaNet/Data/DataFrame.cs @@ -3,7 +3,7 @@ using System; using SiaNet.Engine; using System.Linq; - using NumSharp.Core; + using NumSharp; /// /// Data frame to load data like CSV, images and text in binary format. The instance is then send to Train or Predict method @@ -54,6 +54,12 @@ public virtual void Load(params float[] data) UnderlayingVariable = np.array(data); } + public void Load(float[] data, long[] shape) + { + UnderlayingVariable = np.array(data); + UnderlayingVariable.reshape(BackendUtil.CastShapeInt(shape)); + } + /// /// Gets the underlaying tensor instance. /// @@ -72,7 +78,7 @@ public Array DataArray { get { - return UnderlayingVariable.Data(); + return UnderlayingVariable.Data(); } } @@ -161,7 +167,37 @@ public void Head(int count = 5, string title = "") count = UnderlayingVariable.shape[0]; } + if(!string.IsNullOrWhiteSpace(title)) + { + Console.WriteLine("-----------------{0}----------------", title); + } + Console.WriteLine(UnderlayingVariable[new NDArray(Enumerable.Range(0, count - 1).ToArray())].ToString()); } + + /// + /// Round to nearest integer number + /// + public void Round() + { + } + + public void Max(int? dim = null) + { + if (dim.HasValue) + UnderlayingVariable = UnderlayingVariable.max(dim.Value); + else + UnderlayingVariable = UnderlayingVariable.max(); + } + + public void Min(int? dim = null) + { + UnderlayingVariable = UnderlayingVariable.min(dim); + } + + public void Argmax() + { + UnderlayingVariable = UnderlayingVariable.argmax(); + } } } diff --git a/SiaNet/Data/DataFrame2D.cs b/SiaNet/Data/DataFrame2D.cs index b7485a6..3f6ecba 100644 --- a/SiaNet/Data/DataFrame2D.cs +++ b/SiaNet/Data/DataFrame2D.cs @@ -6,7 +6,7 @@ using CsvHelper; using System.Data; using SiaNet.Engine; - using NumSharp.Core; + using NumSharp; /// /// diff --git a/SiaNet/Data/DataFrameIter.cs b/SiaNet/Data/DataFrameIter.cs index ba2ddcb..e7fa8c0 100644 --- a/SiaNet/Data/DataFrameIter.cs +++ b/SiaNet/Data/DataFrameIter.cs @@ -1,6 +1,6 @@ namespace SiaNet.Data { - using NumSharp.Core; + using NumSharp; using SiaNet.Engine; using System; diff --git a/SiaNet/Data/ImageFrame.cs b/SiaNet/Data/ImageFrame.cs index 1921fcf..d9383d0 100644 --- a/SiaNet/Data/ImageFrame.cs +++ b/SiaNet/Data/ImageFrame.cs @@ -2,7 +2,7 @@ { using System; using System.Drawing; - using NumSharp.Core; + using NumSharp; using OpenCvSharp.Extensions; using SiaNet.Engine; diff --git a/SiaNet/SiaNet.csproj b/SiaNet/SiaNet.csproj index 5b19fe5..2529a3c 100644 --- a/SiaNet/SiaNet.csproj +++ b/SiaNet/SiaNet.csproj @@ -47,10 +47,10 @@ Code well structured, easy to extend if you would like to extend with new layer, - + - + diff --git a/SiaNet/TrainPredict.cs b/SiaNet/TrainPredict.cs index 61aeef2..a428392 100644 --- a/SiaNet/TrainPredict.cs +++ b/SiaNet/TrainPredict.cs @@ -197,7 +197,7 @@ private void RunTrainOnBatch(int i, Tensor x, Tensor y) /// /// The input data frame to run prediction. /// - public Tensor Predict(DataFrame x) + public DataFrame Predict(DataFrame x) { List predictions = new List(); @@ -212,8 +212,10 @@ public Tensor Predict(DataFrame x) } predictions.AddRange(output.ToArray().Cast()); + DataFrame result = new DataFrame(); + result.Load(predictions.ToArray()); - return K.CreateVariable(predictions.ToArray(), output.Shape); + return result; } /// @@ -222,13 +224,12 @@ public Tensor Predict(DataFrame x) /// The input data frame to run prediction. /// Size of the batch. /// - public Tensor Predict(DataFrame x, int batch_size) + public DataFrame Predict(DataFrame x, int batch_size) { DataFrameIter dataFrameIter = new DataFrameIter(x); List predictions = new List(); dataFrameIter.SetBatchSize(batch_size); - long[] outshape = null; - + while (dataFrameIter.Next()) { var data = dataFrameIter.GetBatchX(); @@ -245,7 +246,10 @@ public Tensor Predict(DataFrame x, int batch_size) predictions.AddRange(output.ToArray().Cast()); } - return K.CreateVariable(predictions.ToArray(), outshape); + DataFrame result = new DataFrame(); + result.Load(predictions.ToArray()); + + return result; } ///