Tensor<T> and supporting types, part 3

[tannergooding]

This is a continuation of #103611 and covers the various operation like APIs that do meaningful work with Tensor<T> and supporting types.

API Proposal -- New APIs for Tensor, TensorSpan, and TensorPrimitives

namespace System.Numerics.Tensors;

public static partial class Tensor
{
    public static T Mean<T>(in ROTensorSpan<T> x) where T : IFloatingPoint<T>;

    // The following APIs could have Tensor<bool> and bool versions,
    // following the `*()`, `*All()` and `*Any()` pattern:
    //
    // IBinaryNumber
    // * IsPow2
    // INumberBase
    // * IsCanonical
    // * IsComplexNumber
    // * IsEvenInteger
    // * IsFinite
    // * IsImaginaryNumber
    // * IsInfinity
    // * IsInteger
    // * IsNaN
    // * IsNegative
    // * IsNegativeInfinity
    // * IsNormal
    // * IsOddInteger
    // * IsPositive
    // * IsPositiveInfinity
    // * IsRealNumber
    // * IsSubnormal
    // * IsZero
}

public static partial class TensorPrimitives
{
    public static T Mean<T>(ROSpan<T> x) where T : IFloatingPoint<T>;

    // The following APIs could have Tensor<bool> and bool versions,
    // following the `*()`, `*All()` and `*Any()` pattern:
    //
    // IBinaryNumber
    // * IsPow2
    // INumberBase
    // * IsCanonical
    // * IsComplexNumber
    // * IsEvenInteger
    // * IsFinite
    // * IsImaginaryNumber
    // * IsInfinity
    // * IsInteger
    // * IsNaN
    // * IsNegative
    // * IsNegativeInfinity
    // * IsNormal
    // * IsOddInteger
    // * IsPositive
    // * IsPositiveInfinity
    // * IsRealNumber
    // * IsSubnormal
    // * IsZero
}

API Proposal -- APIs unique to Tensor and TensorSpan

These cover APIs that do not have an equivalent in TensorPrimitives, typically being functionality that is unique to multi-dimensional types:

namespace System.Numerics.Tensors;

public static partial class Tensor
{
    public static ROTensorSpan<T> AsROTensorSpan<T>(this T[]? array, params scoped ROSpan<nint> lengths);
    public static TensorSpan<T> AsTensorSpan<T>(this T[]? array, params scoped ROSpan<nint> lengths);

    public static Tensor<T> Broadcast<T>(in ROTensorSpan<T> x, ROSpan<nint> lengths);

    // This gets the lengths from lengthsSource
    public static Tensor<T> Broadcast<T>(in ROTensorSpan<T> x, in ROTensorSpan<T> lengthsSource);

    public static void BroadcastTo<T>(this Tensor<T> x, in TensorSpan<T> destination);
    public static void BroadcastTo<T>(in this TensorSpan<T> x, in TensorSpan<T> destination);
    public static void BroadcastTo<T>(in this ROTensorSpan<T> x, in TensorSpan<T> destination);

    // The parameter order here feels off. Ideally tensors could be params
    // and we'd have overloads taking 2-3x ROTensorSpan<T>. Maybe we can
    // explode the overloads a bit to make this nicer?
    public static Tensor<T> Concatenate<T>(ROSpan<Tensor<T>> tensors, int axis = 0);
    public static TensorSpan<T> Concatenate<T>(scoped ROSpan<Tensor<T>> tensors, in TensorSpan<T> destination, int axis = 0);

    public static Tensor<T> Create<T>(ROSpan<nint> lengths, bool pinned = false);
    public static Tensor<T> Create<T>(ROSpan<nint> lengths, ROSpan<nint> strides, bool pinned = false);
    public static Tensor<T> Create<T>(T[] values, ROSpan<nint> lengths);
    public static Tensor<T> Create<T>(T[] values, ROSpan<nint> lengths, ROSpan<nint> strides, bool isPinned = false);
    public static Tensor<T> Create<T>(IEnumerable<T> values, ROSpan<nint> lengths);
    public static Tensor<T> Create<T>(IEnumerable<T> values, ROSpan<nint> lengths, ROSpan<nint> strides, bool isPinned = false);

    public static Tensor<T> CreateAndFillGaussianNormalDistribution<T>(params ROSpan<nint> lengths) where T : IFloatingPoint<T>;
    public static Tensor<T> CreateAndFillUniformDistribution<T>(params ROSpan<nint> lengths) where T : IFloatingPoint<T>;

    public static Tensor<T> CreateUninitialized<T>(ROSpan<nint> lengths, bool pinned = false);
    public static Tensor<T> CreateUninitialized<T>(ROSpan<nint> lengths, ROSpan<nint> strides, bool pinned = false);

    public static TensorSpan<T> FillGaussianNormalDistribution<T>(in TensorSpan<T> destination) where T : IFloatingPoint<T>;
    public static TensorSpan<T> FillUniformDistribution<T>(in TensorSpan<T> destination) where T : IFloatingPoint<T>;

    // Should this be some overload of SetSlice or similar?
    public static TensorSpan<T> FilteredUpdate<T>(in this TensorSpan<T> x, scoped in ROTensorSpan<bool> filter, scoped in ROTensorSpan<T> values);
    public static TensorSpan<T> FilteredUpdate<T>(in this TensorSpan<T> x, scoped in ROTensorSpan<bool> filter, T value);

    // Change order of the stride parameters
    public static Tensor<T> Permute<T>(this Tensor<T> x, params ROSpan<int> axis);
    public static TensorSpan<T> Permute<T>(in this TensorSpan<T> x, params scoped ROSpan<int> axis);
    public static ROTensorSpan<T> Permute<T>(in this ROTensorSpan<T> x, params scoped ROSpan<int> axis);

    // Change the lengths
    public static Tensor<T> Reshape<T>(this Tensor<T> x, params ROSpan<nint> lengths);
    public static TensorSpan<T> Reshape<T>(in this TensorSpan<T> x, params scoped ROSpan<nint> lengths);
    public static ROTensorSpan<T> Reshape<T>(in this ROTensorSpan<T> x, params scoped ROSpan<nint> lengths);

    public static Tensor<T> Resize<T>(Tensor<T> x, ROSpan<nint> lengths);

    public static void ResizeTo<T>(this Tensor<T> x, in TensorSpan<nint> destination);
    public static void ResizeTo<T>(in this TensorSpan<T> x, in TensorSpan<nint> destination);
    public static void ResizeTo<T>(in this ROTensorSpan<T> x, in TensorSpan<nint> destination);

    // Should this be expanded to avoid the optional axis parameter?
    public static Tensor<T> Reverse<T>(in ROTensorSpan<T> x, nint axis = -1);
    public static TensorSpan<T> Reverse<T>(scoped in ROTensorSpan<T> x, in TensorSpan<T> destination, nint axis = -1);

    // These are distinct from EqualsAll in that they use IEquatable
    public static bool SequenceEqual<T>(this in ROTensorSpan<T> span, in ROTensorSpan<T> other) where T : IEquatable<T>?;
    public static bool SequenceEqual<T>(this in TensorSpan<T> span, in ROTensorSpan<T> other) where T : IEquatable<T>?;

    // The ordering of these parameters is slightly backwards so params can work for the ranges
    public static Tensor<T> SetSlice<T>(this Tensor<T> tensor, in ROTensorSpan<T> values, params ROSpan<NRange> ranges);
    public static TensorSpan<T> SetSlice<T>(this TensorSpan<T> tensor, scoped in ROTensorSpan<T> values, params scoped ROSpan<NRange> ranges);

    public static Tensor<T>[] Split<T>(in ROTensorSpan<T> x, nint numSplits, nint axis);

    // Remove dimensions of length 1
    public static Tensor<T> Squeeze<T>(this Tensor<T> x, int axis = -1);
    public static TensorSpan<T> Squeeze<T>(in this TensorSpan<T> x, int axis = -1);
    public static ROTensorSpan<T> Squeeze<T>(in this ROTensorSpan<T> x, int axis = -1);

    public static Tensor<T> Stack<T>(ROSpan<Tensor<T>> tensors, int axis = 0);
    public static TensorSpan<T> Stack<T>(scoped ROSpan<Tensor<T>> tensors, in TensorSpan<T> destination, int axis = 0);

    public static string ToString<T>(in this ROTensorSpan<T> span, params ROSpan<nint> maximumLengths);
    public static string ToString<T>(in this TensorSpan<T> span, params ROSpan<nint> maximumLengths);

    public static Tensor<T> Transpose<T>(in ROTensorSpan<T> x);
    public static TensorSpan<T> Transpose<T>(scoped in ROTensorSpan<T> x, in TensorSpan<T> destination);

    public static bool TryBroadcastTo<T>(this Tensor<T> x, in TensorSpan<T> destination);
    public static bool TryBroadcastTo<T>(in this TensorSpan<T> x, in TensorSpan<T> destination);
    public static bool TryBroadcastTo<T>(in this ROTensorSpan<T> x, in TensorSpan<T> destination);

    // Adds dimension of length 1
    public static Tensor<T> Unsqueeze<T>(this Tensor<T> x, int axis);
    public static TensorSpan<T> Unsqueeze<T>(in this TensorSpan<T> x, int axis);
    public static ROTensorSpan<T> Unsqueeze<T>(in this ROTensorSpan<T> x, int axis);
}

[dotnet-policy-service]

Tagging subscribers to this area: @dotnet/area-system-numerics-tensors
See info in area-owners.md if you want to be subscribed.

[Clockwork-Muse]

... Will there be any way to convert between different tensor element types?

I am currently trying to build something that's running an ML model. My image data is of course byte, but the model needs ([0, 1)) float. So I either need to iterate over the spans, or implement an SIMD loop of some sort. Having this built-in would make my life much easier (well, once the 3p libraries adopted this).

[tannergooding]

That was covered in part 2, we have the same general ConvertToChecked, ConvertToTruncating, and ConvertToSaturating APIs that exist in generic math.

[gaviny82]

Does *GaussianNormalDistribution mean the zero mean and unit variance Gaussian/normal distribution? I think either *GaussianDistribution or *NormalDistribution would be enough.

[bartonjs]

Video

• As with the previous review, "RO" is used in lieu of "ReadOnly" because of GH size limits.
• Mean => Average, for consistency with System.Linq.Enumerable
• We agreed to the pattern of just sticking "Any" and "All" as suffixes on the existing methods, even though "IsFooAll" or "IsFooAny" reads a bit strangely.
  • public static bool [T::]IsFoo();
  • public static VectorLike<bool-like> IsFoo(VectorLike<T> foos);
  • public static bool IsFooAny(VectorLike<T> foos);
  • public static bool IsFooAll(VectorLike<T> foos);
• pinned vs isPinned: pinned, to match GC.AllocateArray
• Tensor.Split, numSplits nint => int. (Then got renamed)
• In the tensor class, most of the first parameters got renamed from x to tensor, except for Broadcast, where they're named source.

namespace System.Numerics.Tensors;

public static partial class Tensor
{
    public static T Average<T>(in ROTensorSpan<T> x) where T : IFloatingPoint<T>;

    public static Tensor<bool> IsPow2(in ROTensorSpan<T> x) where T : IBinaryNumber;
    public static TensorSpan<bool> IsPow2(scoped in ROTensorSpan<T> x, in TensorSpan<bool> destination) where T : IBinaryNumber;
    public static bool IsPow2All(in ROTensorSpan<T> x) where T : IBinaryNumber;
    public static bool IsPow2Any(in ROTensorSpan<T> x) where T : IBinaryNumber;

    public static Tensor<bool> IsCanonical(in ROTensorSpan<T> x) where T : INumberBase;
    public static TensorSpan<bool> IsCanonical(scoped in ROTensorSpan<T> x, in TensorSpan<bool> destination) where T : INumberBase;
    public static bool IsCanonicalAll(in ROTensorSpan<T> x) where T : INumberBase;
    public static bool IsCanonicalAny(in ROTensorSpan<T> x) where T : INumberBase;
    // repeat for:
    // * IsComplexNumber
    // * IsEvenInteger
    // * IsFinite
    // * IsImaginaryNumber
    // * IsInfinity
    // * IsInteger
    // * IsNaN
    // * IsNegative
    // * IsNegativeInfinity
    // * IsNormal
    // * IsOddInteger
    // * IsPositive
    // * IsPositiveInfinity
    // * IsRealNumber
    // * IsSubnormal
    // * IsZero
}

public static partial class TensorPrimitives
{
    public static T Average<T>(ROSpan<T> x) where T : IFloatingPoint<T>;

    public static void IsPow2(scoped in ROSpan<T> x, in Span<bool> destination) where T : IBinaryNumber;
    public static bool IsPow2All(in ROSpan<T> x) where T : IBinaryNumber;
    public static bool IsPow2Any(in ROSpan<T> x) where T : IBinaryNumber;

    public static void IsCanonical(scoped in ROSpan<T> x, in Span<bool> destination) where T : INumberBase;
    public static bool IsCanonicalAll(in ROSpan<T> x) where T : INumberBase;
    public static bool IsCanonicalAny(in ROSpan<T> x) where T : INumberBase;
    // repeat for
    // * IsComplexNumber
    // * IsEvenInteger
    // * IsFinite
    // * IsImaginaryNumber
    // * IsInfinity
    // * IsInteger
    // * IsNaN
    // * IsNegative
    // * IsNegativeInfinity
    // * IsNormal
    // * IsOddInteger
    // * IsPositive
    // * IsPositiveInfinity
    // * IsRealNumber
    // * IsSubnormal
    // * IsZero
}

namespace System.Numerics.Tensors;

public static partial class Tensor
{
    public static ROTensorSpan<T> AsReadOnlyTensorSpan<T>(this T[]? array, params scoped ROSpan<nint> lengths);
    public static TensorSpan<T> AsTensorSpan<T>(this T[]? array, params scoped ROSpan<nint> lengths);

    public static Tensor<T> Broadcast<T>(in ROTensorSpan<T> source, ROSpan<nint> lengths);

    // This gets the lengths from lengthsSource
    public static Tensor<T> Broadcast<T>(in ROTensorSpan<T> source, in ROTensorSpan<T> lengthsSource);

    public static void BroadcastTo<T>(this Tensor<T> source, in TensorSpan<T> destination);
    public static void BroadcastTo<T>(in this TensorSpan<T> source, in TensorSpan<T> destination);
    public static void BroadcastTo<T>(in this ROTensorSpan<T> source, in TensorSpan<T> destination);

    public static Tensor<T> Concatenate<T>(params ROSpan<Tensor<T>> tensors);
    public static Tensor<T> ConcatenateOnDimension<T>(int dimension, params ROSpan<Tensor<T>> tensors);
    public static TensorSpan<T> Concatenate<T>(scoped ROSpan<Tensor<T>> tensors, in TensorSpan<T> destination);
    public static TensorSpan<T> ConcatenateOnDimension<T>(int dimension, scoped ROSpan<Tensor<T>> tensors, in TensorSpan<T> destination);

    public static Tensor<T> Create<T>(ROSpan<nint> lengths, bool pinned = false);
    public static Tensor<T> Create<T>(ROSpan<nint> lengths, ROSpan<nint> strides, bool pinned = false);
    public static Tensor<T> Create<T>(T[] values, ROSpan<nint> lengths, bool pinned = false);
    public static Tensor<T> Create<T>(T[] values, ROSpan<nint> lengths, ROSpan<nint> strides, bool pinned = false);
    public static Tensor<T> Create<T>(IEnumerable<T> values, ROSpan<nint> lengths, bool pinned = false);
    public static Tensor<T> Create<T>(IEnumerable<T> values, ROSpan<nint> lengths, ROSpan<nint> strides, bool pinned = false);

    public static Tensor<T> CreateAndFillGaussianNormalDistribution<T>(params ROSpan<nint> lengths) where T : IFloatingPoint<T>;
    public static Tensor<T> CreateAndFillGaussianNormalDistribution<T>(Random random, params ROSpan<nint> lengths) where T : IFloatingPoint<T>;
    public static Tensor<T> CreateAndFillUniformDistribution<T>(params ROSpan<nint> lengths) where T : IFloatingPoint<T>;
    public static Tensor<T> CreateAndFillUniformDistribution<T>(Random random, params ROSpan<nint> lengths) where T : IFloatingPoint<T>;

    public static Tensor<T> CreateUninitialized<T>(ROSpan<nint> lengths, bool pinned = false);
    public static Tensor<T> CreateUninitialized<T>(ROSpan<nint> lengths, ROSpan<nint> strides, bool pinned = false);

    public static TensorSpan<T> FillGaussianNormalDistribution<T>(in TensorSpan<T> destination, Random? random = null) where T : IFloatingPoint<T>;
    public static TensorSpan<T> FillUniformDistribution<T>(in TensorSpan<T> destination, Random? random = null) where T : IFloatingPoint<T>;

    public static TensorSpan<T> FilteredUpdate<T>(in this TensorSpan<T> tensor, scoped in ROTensorSpan<bool> filter, scoped in ROTensorSpan<T> values);
    public static TensorSpan<T> FilteredUpdate<T>(in this TensorSpan<T> tensor, scoped in ROTensorSpan<bool> filter, T value);

    public static Tensor<T> PermuteDimensions<T>(this Tensor<T> tensor, params ROSpan<int> dimensions);
    public static TensorSpan<T> PermuteDimensions<T>(in this TensorSpan<T> tensor, params scoped ROSpan<int> dimensions);
    public static ROTensorSpan<T> PermuteDimensions<T>(in this ROTensorSpan<T> tensor, params scoped ROSpan<int> dimensions);

    public static Tensor<T> Reshape<T>(this Tensor<T> tensor, params ROSpan<nint> lengths);
    public static TensorSpan<T> Reshape<T>(in this TensorSpan<T> tensor, params scoped ROSpan<nint> lengths);
    public static ROTensorSpan<T> Reshape<T>(in this ROTensorSpan<T> tensor, params scoped ROSpan<nint> lengths);

    public static Tensor<T> Resize<T>(Tensor<T> tensor, ROSpan<nint> lengths);

    public static void ResizeTo<T>(this Tensor<T> tensor, in TensorSpan<T> destination);
    public static void ResizeTo<T>(in this TensorSpan<T> tensor, in TensorSpan<T> destination);
    public static void ResizeTo<T>(in this ROTensorSpan<T> tensor, in TensorSpan<T> destination);

    public static Tensor<T> Reverse<T>(in ROTensorSpan<T> tensor);
    public static Tensor<T> ReverseDimension<T>(in ROTensorSpan<T> tensor, int dimension);
    public static TensorSpan<T> Reverse<T>(scoped in ROTensorSpan<T> tensor, in TensorSpan<T> destination);
    public static TensorSpan<T> ReverseDimension<T>(scoped in ROTensorSpan<T> tensor, int dimension, in TensorSpan<T> destination);

    public static bool SequenceEqual<T>(this in ROTensorSpan<T> tensor, in ROTensorSpan<T> other) where T : IEquatable<T>?;
    public static bool SequenceEqual<T>(this in TensorSpan<T> tensor, in ROTensorSpan<T> other) where T : IEquatable<T>?;

    public static Tensor<T> SetSlice<T>(this Tensor<T> tensor, in ROTensorSpan<T> values, params ROSpan<NRange> ranges);
    public static TensorSpan<T> SetSlice<T>(this TensorSpan<T> tensor, scoped in ROTensorSpan<T> values, params scoped ROSpan<NRange> ranges);

    public static Tensor<T>[] Split<T>(in ROTensorSpan<T> tensor, int splitCount, int dimension);

    public static Tensor<T> Squeeze<T>(this Tensor<T> tensor);
    public static Tensor<T> SqueezeDimension<T>(this Tensor<T> tensor, int dimension);
    public static TensorSpan<T> Squeeze<T>(in this TensorSpan<T> tensor);
    public static ROTensorSpan<T> SqueezeDimension<T>(in this ROTensorSpan<T> tensor, int dimension);

    public static Tensor<T> Stack<T>(params ROSpan<Tensor<T>> tensors);
    public static Tensor<T> StackAlongDimension<T>(int dimension, params ROSpan<Tensor<T>> tensors);
    public static TensorSpan<T> Stack<T>(scoped ROSpan<Tensor<T>> tensors, in TensorSpan<T> destination);
    public static TensorSpan<T> StackAlongDimension<T>(scoped ROSpan<Tensor<T>> tensors, in TensorSpan<T> destination, int dimension);

    public static string ToString<T>(in this Tensor<T> tensor, params ROSpan<nint> maximumLengths);
    public static string ToString<T>(in this ROTensorSpan<T> tensor, params ROSpan<nint> maximumLengths);
    public static string ToString<T>(in this TensorSpan<T> tensor, params ROSpan<nint> maximumLengths);

    public static Tensor<T> Transpose<T>(in ROTensorSpan<T> tensor);
    public static TensorSpan<T> Transpose<T>(scoped in ROTensorSpan<T> tensor, in TensorSpan<T> destination);

    public static bool TryBroadcastTo<T>(this Tensor<T> source, in TensorSpan<T> destination);
    public static bool TryBroadcastTo<T>(in this TensorSpan<T> source, in TensorSpan<T> destination);
    public static bool TryBroadcastTo<T>(in this ROTensorSpan<T> source, in TensorSpan<T> destination);

    public static Tensor<T> Unsqueeze<T>(this Tensor<T> tensor, int dimension);
    public static TensorSpan<T> Unsqueeze<T>(in this TensorSpan<T> tensor, int dimension);
    public static ROTensorSpan<T> Unsqueeze<T>(in this ROTensorSpan<T> tensor, int dimension);
}