From 5196d1661c1ab0b66e793c86cbdba64a6e4250e5 Mon Sep 17 00:00:00 2001 From: Ian Hawley Date: Thu, 27 Jun 2024 19:31:31 +0100 Subject: [PATCH 01/18] Removed obsolete code and V1 serializer --- .../Operations/Operations.Receive.Obsolete.cs | 533 ------------- .../Operations/Operations.Send.Obsolete.cs | 241 ------ .../Api/Operations/Operations.Serialize.cs | 107 --- src/Speckle.Core/Api/Operations/Operations.cs | 21 - src/Speckle.Core/Models/Base.cs | 19 - .../Serialisation/BaseObjectSerializer.cs | 719 ------------------ src/Speckle.Objects/SpeckleSchemaInfo.cs | 6 + .../Api/Operations/SendReceiveLocal.cs | 55 +- 8 files changed, 7 insertions(+), 1694 deletions(-) delete mode 100644 src/Speckle.Core/Api/Operations/Operations.Receive.Obsolete.cs delete mode 100644 src/Speckle.Core/Api/Operations/Operations.Send.Obsolete.cs delete mode 100644 src/Speckle.Core/Serialisation/BaseObjectSerializer.cs create mode 100644 src/Speckle.Objects/SpeckleSchemaInfo.cs diff --git a/src/Speckle.Core/Api/Operations/Operations.Receive.Obsolete.cs b/src/Speckle.Core/Api/Operations/Operations.Receive.Obsolete.cs deleted file mode 100644 index 5c486aa..0000000 --- a/src/Speckle.Core/Api/Operations/Operations.Receive.Obsolete.cs +++ /dev/null @@ -1,533 +0,0 @@ -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Serilog.Context; -using Speckle.Core.Logging; -using Speckle.Core.Models; -using Speckle.Core.Serialisation; -using Speckle.Core.Transports; -using Speckle.Newtonsoft.Json; - -namespace Speckle.Core.Api; - -#pragma warning disable CA1068, IDE1006 - -[Obsolete("Serializer v1 is deprecated")] -public enum SerializerVersion -{ - V1, - V2 -} - -public static partial class Operations -{ - private const string RECEIVE_DEPRECATION_MESSAGE = """ - This method overload is obsolete, consider using a non-obsolete overload. - 1.SerializerVersion selection will no longer be supported going foward (serializer v1 is now deprecated). - 2.Use of disposeTransports will no longer be supported going forward (you should dispose your own transports). - 3 OnErrorAction is no longer used (instead functions with throw exceptions for consistancy and clear stack trace) - """; - - /// - /// - [Obsolete(RECEIVE_DEPRECATION_MESSAGE)] - public static Task Receive( - string objectId, - CancellationToken cancellationToken, - ITransport? remoteTransport, - ITransport? localTransport, - Action>? onProgressAction, - Action? onErrorAction, - Action? onTotalChildrenCountKnown, - bool disposeTransports - ) - { - return Receive( - objectId, - cancellationToken, - remoteTransport, - localTransport, - onProgressAction, - onErrorAction, - onTotalChildrenCountKnown, - disposeTransports, - SerializerVersion.V2 - ); - } - - /// - /// - [Obsolete(RECEIVE_DEPRECATION_MESSAGE)] - public static Task Receive( - string objectId, - CancellationToken cancellationToken, - ITransport? remoteTransport, - Action>? onProgressAction, - Action? onErrorAction, - Action? onTotalChildrenCountKnown, - bool disposeTransports, - SerializerVersion serializerVersion - ) - { - return Receive( - objectId, - cancellationToken, - remoteTransport, - null, - onProgressAction, - onErrorAction, - onTotalChildrenCountKnown, - disposeTransports, - serializerVersion - ); - } - - /// - /// - [Obsolete(RECEIVE_DEPRECATION_MESSAGE)] - public static Task Receive( - string objectId, - CancellationToken cancellationToken, - ITransport? remoteTransport, - Action>? onProgressAction, - Action? onErrorAction, - Action? onTotalChildrenCountKnown, - bool disposeTransports - ) - { - return Receive( - objectId, - cancellationToken, - remoteTransport, - null, - onProgressAction, - onErrorAction, - onTotalChildrenCountKnown, - disposeTransports, - SerializerVersion.V2 - ); - } - - /// - /// - [Obsolete(RECEIVE_DEPRECATION_MESSAGE)] - public static Task Receive( - string objectId, - CancellationToken cancellationToken, - ITransport? remoteTransport, - bool disposeTransports - ) - { - return Receive( - objectId, - cancellationToken, - remoteTransport, - null, - null, - null, - null, - disposeTransports, - SerializerVersion.V2 - ); - } - - /// - /// - [Obsolete(RECEIVE_DEPRECATION_MESSAGE)] - public static Task Receive( - string objectId, - ITransport? remoteTransport, - ITransport? localTransport, - Action>? onProgressAction, - Action? onErrorAction, - Action? onTotalChildrenCountKnown, - bool disposeTransports, - SerializerVersion serializerVersion - ) - { - return Receive( - objectId, - CancellationToken.None, - remoteTransport, - localTransport, - onProgressAction, - onErrorAction, - onTotalChildrenCountKnown, - disposeTransports, - serializerVersion - ); - } - - /// - /// - [Obsolete(RECEIVE_DEPRECATION_MESSAGE)] - public static Task Receive( - string objectId, - ITransport? remoteTransport, - ITransport? localTransport, - Action>? onProgressAction, - Action? onErrorAction, - Action? onTotalChildrenCountKnown, - bool disposeTransports - ) - { - return Receive( - objectId, - CancellationToken.None, - remoteTransport, - localTransport, - onProgressAction, - onErrorAction, - onTotalChildrenCountKnown, - disposeTransports, - SerializerVersion.V2 - ); - } - - /// - /// - [Obsolete(RECEIVE_DEPRECATION_MESSAGE)] - public static Task Receive( - string objectId, - ITransport? remoteTransport, - Action>? onProgressAction, - Action? onErrorAction, - Action? onTotalChildrenCountKnown, - bool disposeTransports - ) - { - return Receive( - objectId, - CancellationToken.None, - remoteTransport, - null, - onProgressAction, - onErrorAction, - onTotalChildrenCountKnown, - disposeTransports, - SerializerVersion.V2 - ); - } - - /// - /// - [Obsolete(RECEIVE_DEPRECATION_MESSAGE)] - public static Task Receive( - string objectId, - ITransport? remoteTransport, - ITransport? localTransport, - bool disposeTransports - ) - { - return Receive( - objectId, - CancellationToken.None, - remoteTransport, - localTransport, - null, - null, - null, - disposeTransports, - SerializerVersion.V2 - ); - } - - /// - /// - [Obsolete(RECEIVE_DEPRECATION_MESSAGE)] - public static Task Receive(string objectId, ITransport? remoteTransport, bool disposeTransports) - { - return Receive( - objectId, - CancellationToken.None, - remoteTransport, - null, - null, - null, - null, - disposeTransports, - SerializerVersion.V2 - ); - } - - /// - /// - [Obsolete(RECEIVE_DEPRECATION_MESSAGE)] - public static Task Receive(string objectId, bool disposeTransports) - { - return Receive( - objectId, - CancellationToken.None, - null, - null, - null, - null, - null, - disposeTransports, - SerializerVersion.V2 - ); - } - - /// - /// - [Obsolete(RECEIVE_DEPRECATION_MESSAGE)] - public static Task Receive( - string objectId, - ITransport? remoteTransport, - ITransport? localTransport, - Action? onErrorAction - ) - { - return Receive( - objectId, - default, - remoteTransport, - localTransport, - null, - onErrorAction, - null, - false, - SerializerVersion.V2 - ); - } - - /// - /// - [Obsolete(RECEIVE_DEPRECATION_MESSAGE)] - public static Task Receive( - string objectId, - ITransport? remoteTransport, - Action? onErrorAction - ) - { - return Receive(objectId, default, remoteTransport, null, null, onErrorAction, null, false, SerializerVersion.V2); - } - - /// - /// Receives an object from a transport. - /// - /// - /// This overload is deprecated. You should consider using - /// - ///
- /// The new overload no longer support switching as v1 is now deprecated. - ///
- /// We also no longer offer the option to . - /// You should instead handle disposal yourself - /// using conventional mechanisms like the using keyword or try finally block
- ///
- /// This function overload will be kept around for several releases, but will eventually be removed. - /// - /// - /// The transport to receive from. - /// Leave null to use the default cache. - /// Action invoked on progress iterations. - /// Action invoked on internal errors. - /// Action invoked once the total count of objects is known. - /// - [Obsolete(RECEIVE_DEPRECATION_MESSAGE)] - public static async Task Receive( - string objectId, - CancellationToken cancellationToken, - ITransport? remoteTransport, - ITransport? localTransport, - Action>? onProgressAction, - Action? onErrorAction, - Action? onTotalChildrenCountKnown, - bool disposeTransports, - SerializerVersion serializerVersion - ) - { - var hasUserProvidedLocalTransport = localTransport != null; - localTransport ??= new SQLiteTransport(); - using (LogContext.PushProperty("remoteTransportContext", remoteTransport?.TransportContext)) - using (LogContext.PushProperty("localTransportContext", localTransport.TransportContext)) - using (LogContext.PushProperty("objectId", objectId)) - { - var timer = Stopwatch.StartNew(); - SpeckleLog.Logger.Information( - "Starting receive {objectId} from transports {localTransport} / {remoteTransport}", - objectId, - localTransport.TransportName, - remoteTransport?.TransportName - ); - - BaseObjectSerializer? serializer = null; - JsonSerializerSettings? settings = null; - BaseObjectDeserializerV2? serializerV2 = null; - if (serializerVersion == SerializerVersion.V1) - { - (serializer, settings) = GetSerializerInstance(); - } - else - { - serializerV2 = new BaseObjectDeserializerV2(); - } - - var internalProgressAction = GetInternalProgressAction(onProgressAction); - - localTransport.OnProgressAction = internalProgressAction; - localTransport.CancellationToken = cancellationToken; - - if (serializerVersion == SerializerVersion.V1) - { - serializer!.ReadTransport = localTransport; - serializer.OnProgressAction = internalProgressAction; - serializer.OnErrorAction = onErrorAction; - serializer.CancellationToken = cancellationToken; - } - else - { - serializerV2!.ReadTransport = localTransport; - serializerV2.OnProgressAction = internalProgressAction; - serializerV2.OnErrorAction = onErrorAction; - serializerV2.CancellationToken = cancellationToken; - if (remoteTransport is IBlobCapableTransport t) - { - serializerV2.BlobStorageFolder = t.BlobStorageFolder; - } - } - - // First we try and get the object from the local transport. If it's there, we assume all its children are there, and proceed with deserialisation. - // This assumption is hard-wired into the SDK. Read below. - var objString = localTransport.GetObject(objectId); - - if (objString != null) - { - // Shoot out the total children count - var partial = JsonConvert.DeserializeObject(objString); - if (partial == null) - { - throw new SpeckleDeserializeException( - $"Failed to deserialize {nameof(objString)} into {nameof(Placeholder)}" - ); - } - - if (partial.__closure != null) - { - onTotalChildrenCountKnown?.Invoke(partial.__closure.Count); - } - - Base? localRes = DeserializeStringToBase(serializerVersion, objString, settings, serializerV2); - - if ((disposeTransports || !hasUserProvidedLocalTransport) && localTransport is IDisposable dispLocal) - { - dispLocal.Dispose(); - } - - if (disposeTransports && remoteTransport != null && remoteTransport is IDisposable dispRemote) - { - dispRemote.Dispose(); - } - - timer.Stop(); - SpeckleLog - .Logger.ForContext("deserializerElapsed", serializerV2?.Elapsed) - .ForContext( - "transportElapsedBreakdown", - new[] { localTransport, remoteTransport } - .Where(t => t != null) - .ToDictionary(t => t!.TransportName, t => t!.Elapsed) - ) - .Information( - "Finished receiving {objectId} from {source} in {elapsed} seconds", - objectId, - localTransport.TransportName, - timer.Elapsed.TotalSeconds - ); - return localRes; - } - - if (remoteTransport == null) - { - var ex = new SpeckleException( - $"Could not find specified object using the local transport {localTransport.TransportName}, and you didn't provide a fallback remote from which to pull it." - ); - - SpeckleLog.Logger.Error(ex, "Cannot receive object from the given transports {exceptionMessage}", ex.Message); - throw ex; - } - - // If we've reached this stage, it means that we didn't get a local transport hit on our object, so we will proceed to get it from the provided remote transport. - // This is done by copying itself and all its children from the remote transport into the local one. - remoteTransport.OnProgressAction = internalProgressAction; - remoteTransport.CancellationToken = cancellationToken; - - SpeckleLog.Logger.Debug( - "Cannot find object {objectId} in the local transport, hitting remote {transportName}", - remoteTransport.TransportName - ); - objString = await remoteTransport - .CopyObjectAndChildren(objectId, localTransport, onTotalChildrenCountKnown) - .ConfigureAwait(false); - - // Wait for the local transport to finish "writing" - in this case, it signifies that the remote transport has done pushing copying objects into it. (TODO: I can see some scenarios where latency can screw things up, and we should rather wait on the remote transport). - await localTransport.WriteComplete().ConfigureAwait(false); - - // Proceed to deserialise the object, now safely knowing that all its children are present in the local (fast) transport. - - Base? res = DeserializeStringToBase(serializerVersion, objString, settings, serializerV2); - if ((disposeTransports || !hasUserProvidedLocalTransport) && localTransport is IDisposable dl) - { - dl.Dispose(); - } - - if (disposeTransports && remoteTransport is IDisposable dr) - { - dr.Dispose(); - } - - SpeckleLog - .Logger.ForContext("deserializerElapsed", serializerV2?.Elapsed) - .ForContext( - "transportElapsedBreakdown", - new[] { localTransport, remoteTransport } - .Where(t => t != null) - .ToDictionary(t => t.TransportName, t => t.Elapsed) - ) - .Information( - "Finished receiving {objectId} from {source} in {elapsed} seconds", - objectId, - remoteTransport.TransportName, - timer.Elapsed.TotalSeconds - ); - return res; - - // Summary: - // Basically, receiving an object (and all its subchildren) operates with two transports, one that is potentially slow, and one that is fast. - // The fast transport ("localTransport") is used syncronously inside the deserialisation routine to get the value of nested references and set them. The slow transport ("remoteTransport") is used to get the raw data and populate the local transport with all necessary data for a successful deserialisation of the object. - // Note: if properly implemented, there is no hard distinction between what is a local or remote transport; it's still just a transport. So, for example, if you want to receive an object without actually writing it first to a local transport, you can just pass a Server/S3 transport as a local transport. - // This is not reccommended, but shows what you can do. Another tidbit: the local transport does not need to be disk-bound; it can easily be an in memory transport. In memory transports are the fastest ones, but they're of limited use for more - } - } - - [Obsolete("Serializer v1 is deprecated, use other overload(s)")] - private static Base? DeserializeStringToBase( - SerializerVersion serializerVersion, - string objString, - JsonSerializerSettings? settings, - BaseObjectDeserializerV2? serializerV2 - ) - { - if (serializerVersion == SerializerVersion.V1) - { - return JsonConvert.DeserializeObject(objString, settings); - } - else - { - return serializerV2!.Deserialize(objString); - } - } -} - -[Obsolete("Use " + nameof(TransportHelpers.Placeholder))] -internal sealed class Placeholder -{ - public Dictionary? __closure { get; set; } = new(); -} - -#pragma warning restore CA1068, IDE1006 diff --git a/src/Speckle.Core/Api/Operations/Operations.Send.Obsolete.cs b/src/Speckle.Core/Api/Operations/Operations.Send.Obsolete.cs deleted file mode 100644 index 24cfd31..0000000 --- a/src/Speckle.Core/Api/Operations/Operations.Send.Obsolete.cs +++ /dev/null @@ -1,241 +0,0 @@ -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Serilog.Context; -using Speckle.Core.Logging; -using Speckle.Core.Models; -using Speckle.Core.Serialisation; -using Speckle.Core.Transports; -using Speckle.Newtonsoft.Json; -using Speckle.Newtonsoft.Json.Linq; - -namespace Speckle.Core.Api; - -public static partial class Operations -{ - private const string DEPRECATION_NOTICE = """ - This Send overload has been replaced by an overload with fewer function arguments. - We are no longer supporting SerializerV1, OnErrorAction, or handling disposal of transports. - Consider switching one of the other send overloads instead. - This function will be kept around for several releases, but will eventually be removed. - """; - - /// - [Obsolete("This overload has been deprecated along with serializer v1. Use other Send overloads instead.")] - [SuppressMessage("Naming", "CA1720:Identifier contains type name")] - public static Task Send(Base @object) => Send(@object, CancellationToken.None); - - /// - [Obsolete("This overload has been deprecated along with serializer v1. Use other Send overloads instead.")] - [SuppressMessage("Naming", "CA1720:Identifier contains type name")] - public static Task Send( - Base @object, - List? transports, - bool useDefaultCache, - Action>? onProgressAction, - Action? onErrorAction, - bool disposeTransports, - SerializerVersion serializerVersion = SerializerVersion.V2 - ) => - Send( - @object, - CancellationToken.None, - transports, - useDefaultCache, - onProgressAction, - onErrorAction, - disposeTransports, - serializerVersion - ); - - /// - [Obsolete("This overload has been deprecated along with serializer v1. Use other Send overloads instead.")] - [SuppressMessage("Naming", "CA1720:Identifier contains type name")] - public static Task Send( - Base @object, - List? transports, - bool useDefaultCache, - bool disposeTransports, - SerializerVersion serializerVersion = SerializerVersion.V2 - ) => - Send( - @object, - CancellationToken.None, - transports, - useDefaultCache, - null, - null, - disposeTransports, - serializerVersion - ); - - /// - [Obsolete("This overload has been deprecated along with serializer v1. Use other Send overloads instead.")] - [SuppressMessage("Naming", "CA1720:Identifier contains type name")] - public static Task Send( - Base @object, - bool disposeTransports, - SerializerVersion serializerVersion = SerializerVersion.V2 - ) => Send(@object, CancellationToken.None, null, true, null, null, disposeTransports, serializerVersion); - - /// - /// Sends an object via the provided transports. Defaults to the local cache. - /// - /// - /// This overload is deprecated. You should consider using - ///
- ///
or - ///
- ///
- /// These new overloads no longer support switching as v1 is now deprecated. - ///
- /// We also no longer offer the option to . - /// You should instead handle disposal yourself - /// using conventional mechanisms like the using keyword.
- ///
- /// This function overload will be kept around for several releases, but will eventually be removed. - /// - /// The object you want to send. - /// A cancellation token that can be used by other objects or threads to send notice of cancellation. - /// Where you want to send them. - /// Toggle for the default cache. If set to false, it will only send to the provided transports. - /// Action that gets triggered on every progress tick (keeps track of all transports). - /// Use this to capture and handle any errors from within the transports. - /// - /// - /// The id (hash) of the object. - [SuppressMessage("Naming", "CA1720:Identifier contains type name")] - [Obsolete(DEPRECATION_NOTICE)] - public static async Task Send( - Base @object, - CancellationToken cancellationToken, - List? transports = null, - bool useDefaultCache = true, - Action>? onProgressAction = null, - Action? onErrorAction = null, - bool disposeTransports = false, - SerializerVersion serializerVersion = SerializerVersion.V2 - ) - { - transports ??= new List(); - using var sqLiteTransport = new SQLiteTransport { TransportName = "LC" }; - - if (transports.Count == 0 && useDefaultCache == false) - { - throw new ArgumentException( - "You need to provide at least one transport: cannot send with an empty transport list and no default cache.", - nameof(transports) - ); - } - - if (useDefaultCache) - { - transports.Insert(0, sqLiteTransport); - } - - var transportContext = transports.ToDictionary(t => t.TransportName, t => t.TransportContext); - - // make sure all logs in the operation have the proper context - using (LogContext.PushProperty("transportContext", transportContext)) - using (LogContext.PushProperty("correlationId", Guid.NewGuid().ToString())) - { - var sendTimer = Stopwatch.StartNew(); - SpeckleLog.Logger.Information("Starting send operation"); - - var internalProgressAction = GetInternalProgressAction(onProgressAction); - - BaseObjectSerializer? serializer = null; - JsonSerializerSettings? settings = null; - BaseObjectSerializerV2? serializerV2 = null; - if (serializerVersion == SerializerVersion.V1) - { - (serializer, settings) = GetSerializerInstance(); - serializer.WriteTransports = transports; - serializer!.OnProgressAction = internalProgressAction; - serializer.CancellationToken = cancellationToken; - serializer.OnErrorAction = onErrorAction; - } - else - { - serializerV2 = new BaseObjectSerializerV2(transports, internalProgressAction, false, cancellationToken); - } - - foreach (var t in transports) - { - t.OnProgressAction = internalProgressAction; - t.CancellationToken = cancellationToken; - t.BeginWrite(); - } - - string obj; - List transportAwaits; - if (serializerVersion == SerializerVersion.V1) - { - obj = JsonConvert.SerializeObject(@object, settings); - transportAwaits = serializer!.WriteTransports.Select(t => t.WriteComplete()).ToList(); - } - else - { - obj = serializerV2!.Serialize(@object); - transportAwaits = serializerV2.WriteTransports.Select(t => t.WriteComplete()).ToList(); - } - - if (cancellationToken.IsCancellationRequested) - { - SpeckleLog.Logger.Information( - "Send operation cancelled after {elapsed} seconds", - sendTimer.Elapsed.TotalSeconds - ); - cancellationToken.ThrowIfCancellationRequested(); - } - - await Task.WhenAll(transportAwaits).ConfigureAwait(false); - - foreach (var t in transports) - { - t.EndWrite(); - if (useDefaultCache && t is SQLiteTransport lc && lc.TransportName == "LC") - { - lc.Dispose(); - continue; - } - if (disposeTransports && t is IDisposable disp) - { - disp.Dispose(); - } - } - - if (cancellationToken.IsCancellationRequested) - { - SpeckleLog.Logger.Information("Send operation cancelled after {elapsed}", sendTimer.Elapsed.TotalSeconds); - cancellationToken.ThrowIfCancellationRequested(); - } - - var idToken = JObject.Parse(obj).GetValue("id"); - if (idToken == null) - { - throw new SpeckleException("Failed to get id of serialized object"); - } - - var hash = idToken.ToString(); - - sendTimer.Stop(); - SpeckleLog - .Logger.ForContext("transportElapsedBreakdown", transports.ToDictionary(t => t.TransportName, t => t.Elapsed)) - .ForContext("note", "the elapsed summary doesn't need to add up to the total elapsed... Threading magic...") - .ForContext("serializerElapsed", serializerV2?.Elapsed) - .Information( - "Finished sending {objectCount} objects after {elapsed}, result {objectId}", - transports.Max(t => t.SavedObjectCount), - sendTimer.Elapsed.TotalSeconds, - hash - ); - return hash; - } - } -} diff --git a/src/Speckle.Core/Api/Operations/Operations.Serialize.cs b/src/Speckle.Core/Api/Operations/Operations.Serialize.cs index bd3bea9..6fa3d89 100644 --- a/src/Speckle.Core/Api/Operations/Operations.Serialize.cs +++ b/src/Speckle.Core/Api/Operations/Operations.Serialize.cs @@ -1,7 +1,3 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Threading; using Speckle.Core.Logging; using Speckle.Core.Models; using Speckle.Core.Serialisation; @@ -45,107 +41,4 @@ public static Base Deserialize(string value, CancellationToken cancellationToken var deserializer = new BaseObjectDeserializerV2 { CancellationToken = cancellationToken }; return deserializer.Deserialize(value); } - - #region obsolete - - [Obsolete("Serializer v1 is deprecated, use other overload(s)")] - public static string Serialize( - Base value, - SerializerVersion serializerVersion, - CancellationToken cancellationToken = default - ) - { - if (serializerVersion == SerializerVersion.V1) - { - var (serializer, settings) = GetSerializerInstance(); - serializer.CancellationToken = cancellationToken; - - return JsonConvert.SerializeObject(value, settings); - } - else - { - return Serialize(value, cancellationToken); - } - } - - [Obsolete("Serializer v1 is deprecated, use other overload(s)")] - public static Base Deserialize( - string value, - SerializerVersion serializerVersion, - CancellationToken cancellationToken = default - ) - { - if (serializerVersion == SerializerVersion.V1) - { - var (serializer, settings) = GetSerializerInstance(); - serializer.CancellationToken = cancellationToken; - var ret = JsonConvert.DeserializeObject(value, settings); - return ret ?? throw new SpeckleException($"{nameof(value)} failed to deserialize to a {nameof(Base)} object"); - } - - return Deserialize(value, cancellationToken); - } - - [Obsolete("Please use the Deserialize(string value) function.", true)] - public static List DeserializeArray( - string objectArr, - SerializerVersion serializerVersion = SerializerVersion.V2 - ) - { - throw new NotImplementedException(); - } - - [Obsolete( - "Please use the Deserialize(Base @object) function. This function will be removed in later versions.", - true - )] - public static Dictionary DeserializeDictionary(string dictionary) - { - throw new NotImplementedException(); - } - - [Obsolete("Use overload that takes cancellation token last")] - [SuppressMessage("Naming", "CA1720:Identifier contains type name")] - public static Base Deserialize( - string @object, - CancellationToken cancellationToken, - SerializerVersion serializerVersion = SerializerVersion.V2 - ) - { - return Deserialize(@object, serializerVersion, cancellationToken); - } - - [Obsolete("Use overload that takes cancellation token last")] - [SuppressMessage("Naming", "CA1720:Identifier contains type name")] - public static string Serialize( - Base @object, - CancellationToken cancellationToken, - SerializerVersion serializerVersion = SerializerVersion.V2 - ) - { - return Serialize(@object, serializerVersion, cancellationToken); - } - - /// - /// Serializes a list of objects. Note: if you want to save and persist objects to speckle, please use any of the "Send" methods. - /// - /// - /// - [Obsolete("Please use the Serialize(Base value) function. This function will be removed in later versions.", true)] - public static string Serialize(List objects) - { - throw new NotImplementedException(); - } - - /// - /// Serializes a list of objects. Note: if you want to save and persist objects to speckle, please use any of the "Send" methods. - /// - /// - /// - [Obsolete("Please use the Serialize(Base value) function. This function will be removed in later versions.")] - public static string Serialize(Dictionary objects) - { - throw new NotImplementedException(); - } - #endregion } diff --git a/src/Speckle.Core/Api/Operations/Operations.cs b/src/Speckle.Core/Api/Operations/Operations.cs index 83e153b..67e61b7 100644 --- a/src/Speckle.Core/Api/Operations/Operations.cs +++ b/src/Speckle.Core/Api/Operations/Operations.cs @@ -1,6 +1,4 @@ -using System; using System.Collections.Concurrent; -using System.Collections.Generic; using Speckle.Core.Serialisation; using Speckle.Newtonsoft.Json; using Speckle.Newtonsoft.Json.Serialization; @@ -14,25 +12,6 @@ namespace Speckle.Core.Api; /// public static partial class Operations { - /// - /// Convenience method to instantiate an instance of the default object serializer and settings pre-populated with it. - /// - [Obsolete("V1 Serializer is deprecated. Use " + nameof(BaseObjectSerializerV2))] - public static (BaseObjectSerializer, JsonSerializerSettings) GetSerializerInstance() - { - var serializer = new BaseObjectSerializer(); - var settings = new JsonSerializerSettings - { - NullValueHandling = NullValueHandling.Ignore, - ContractResolver = new CamelCasePropertyNamesContractResolver(), - Formatting = Formatting.None, - ReferenceLoopHandling = ReferenceLoopHandling.Ignore, - Converters = new List { serializer } - }; - - return (serializer, settings); - } - /// /// Factory for progress actions used internally inside send and receive methods. /// diff --git a/src/Speckle.Core/Models/Base.cs b/src/Speckle.Core/Models/Base.cs index 84d8403..15aa29b 100644 --- a/src/Speckle.Core/Models/Base.cs +++ b/src/Speckle.Core/Models/Base.cs @@ -286,23 +286,4 @@ public Base ShallowCopy() return myDuplicate; } - - #region Obsolete - /// - [Obsolete("Serializer v1 is deprecated, use other overload(s)", true)] - public string GetId(SerializerVersion serializerVersion) - { - return GetId(false, serializerVersion); - } - - /// - [Obsolete("Serializer v1 is deprecated, use other overload(s)", true)] - public string GetId(bool decompose, SerializerVersion serializerVersion) - { - throw new NotImplementedException( - "Overload has been deprecated along with SerializerV1, use other overload (uses SerializerV2)" - ); - } - - #endregion } diff --git a/src/Speckle.Core/Serialisation/BaseObjectSerializer.cs b/src/Speckle.Core/Serialisation/BaseObjectSerializer.cs deleted file mode 100644 index 418df0b..0000000 --- a/src/Speckle.Core/Serialisation/BaseObjectSerializer.cs +++ /dev/null @@ -1,719 +0,0 @@ -#nullable disable -using System; -using System.Collections; -using System.Collections.Generic; -using System.Runtime.Serialization; -using System.Threading; -using Speckle.Core.Helpers; -using Speckle.Core.Logging; -using Speckle.Core.Models; -using Speckle.Core.Serialisation.SerializationUtilities; -using Speckle.Core.Transports; -using Speckle.Newtonsoft.Json; -using Speckle.Newtonsoft.Json.Linq; -using Speckle.Newtonsoft.Json.Serialization; -using Utilities = Speckle.Core.Models.Utilities; - -// ReSharper disable InconsistentNaming -// ReSharper disable UseNegatedPatternInIsExpression -#pragma warning disable IDE0075, IDE1006, IDE0083, CA1051, CA1502, CA1854 - -namespace Speckle.Core.Serialisation; - -/// -/// Json converter that handles base speckle objects. Enables detachment and -/// simultaneous transport (persistence) of objects. -/// -[Obsolete("Use " + nameof(BaseObjectSerializerV2))] -public class BaseObjectSerializer : JsonConverter -{ - /// - /// Property that describes the type of the object. - /// - public string TypeDiscriminator = "speckle_type"; - - public BaseObjectSerializer() - { - ResetAndInitialize(); - } - - public CancellationToken CancellationToken { get; set; } - - /// - /// The sync transport. This transport will be used synchronously. - /// - public ITransport ReadTransport { get; set; } - - /// - /// List of transports to write to. - /// - public List WriteTransports { get; set; } = new(); - - public override bool CanWrite => true; - - public override bool CanRead => true; - - public Action OnProgressAction { get; set; } - - public Action OnErrorAction { get; set; } - - /// - /// Reinitializes the lineage, and other variables that get used during the - /// json writing process. - /// - public void ResetAndInitialize() - { - DetachLineage = new List(); - Lineage = new List(); - RefMinDepthTracker = new Dictionary>(); - OnProgressAction = null; - TotalProcessedCount = 0; - } - - public override bool CanConvert(Type objectType) - { - return true; - } - - #region Read Json - - public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) - { - if (CancellationToken.IsCancellationRequested) - { - return null; // Check for cancellation - } - - if (reader.TokenType == JsonToken.Null) - { - return null; - } - - // Check if we passed in an array, rather than an object. - // TODO: Test the following branch. It's not used anywhere at the moment, and the default serializer prevents it from - // ever being used (only allows single object serialization) - if (reader.TokenType == JsonToken.StartArray) - { - var list = new List(); - var jarr = JArray.Load(reader); - - foreach (var val in jarr) - { - if (CancellationToken.IsCancellationRequested) - { - return null; // Check for cancellation - } - - var whatever = BaseObjectSerializationUtilities.HandleValue(val, serializer, CancellationToken); - list.Add(whatever as Base); - } - return list; - } - - if (CancellationToken.IsCancellationRequested) - { - return null; // Check for cancellation - } - - var jObject = JObject.Load(reader); - - if (jObject == null) - { - return null; - } - - var objType = jObject.GetValue(TypeDiscriminator); - - // Assume dictionary! - if (objType == null) - { - var dict = new Dictionary(); - - foreach (var val in jObject) - { - if (CancellationToken.IsCancellationRequested) - { - return null; // Check for cancellation - } - - dict[val.Key] = BaseObjectSerializationUtilities.HandleValue(val.Value, serializer, CancellationToken); - } - return dict; - } - - if (CancellationToken.IsCancellationRequested) - { - return null; // Check for cancellation - } - - var discriminator = objType.Value(); - - // Check for references. - if (discriminator == "reference") - { - var id = jObject.GetValue("referencedId").Value(); - - string str = - ReadTransport != null - ? ReadTransport.GetObject(id) - : throw new SpeckleException("Cannot resolve reference, no transport is defined."); - - if (str != null && !string.IsNullOrEmpty(str)) - { - jObject = JObject.Parse(str); - discriminator = jObject.GetValue(TypeDiscriminator).Value(); - } - else - { - throw new SpeckleException("Cannot resolve reference. The provided transport could not find it."); - } - } - - var type = BaseObjectSerializationUtilities.GetType(discriminator); - var obj = existingValue ?? Activator.CreateInstance(type); - - var contract = (JsonDynamicContract)serializer.ContractResolver.ResolveContract(type); - var used = new HashSet(); - - // remove unsettable properties - jObject.Remove(TypeDiscriminator); - jObject.Remove("__closure"); - - if (CancellationToken.IsCancellationRequested) - { - return null; // Check for cancellation - } - - foreach (var jProperty in jObject.Properties()) - { - if (CancellationToken.IsCancellationRequested) - { - return null; // Check for cancellation - } - - if (used.Contains(jProperty.Name)) - { - continue; - } - - used.Add(jProperty.Name); - - // first attempt to find a settable property, otherwise fall back to a dynamic set without type - JsonProperty property = contract.Properties.GetClosestMatchProperty(jProperty.Name); - - if (property != null && property.Writable) - { - if (type == typeof(Abstract) && property.PropertyName == "base") - { - var propertyValue = BaseObjectSerializationUtilities.HandleAbstractOriginalValue( - jProperty.Value, - ((JValue)jObject.GetValue("assemblyQualifiedName")).Value as string - ); - property.ValueProvider.SetValue(obj, propertyValue); - } - else - { - var val = BaseObjectSerializationUtilities.HandleValue( - jProperty.Value, - serializer, - CancellationToken, - property - ); - property.ValueProvider.SetValue(obj, val); - } - } - else - { - // dynamic properties - CallSiteCache.SetValue( - jProperty.Name, - obj, - BaseObjectSerializationUtilities.HandleValue(jProperty.Value, serializer, CancellationToken) - ); - } - } - - if (CancellationToken.IsCancellationRequested) - { - return null; // Check for cancellation - } - - TotalProcessedCount++; - OnProgressAction?.Invoke("DS", 1); - - foreach (var callback in contract.OnDeserializedCallbacks) - { - callback(obj, serializer.Context); - } - - return obj; - } - - #endregion - - #region Write Json Helper Properties - - /// - /// Keeps track of wether current property pointer is marked for detachment. - /// - private List DetachLineage { get; set; } - - /// - /// Keeps track of the hash chain through the object tree. - /// - private List Lineage { get; set; } - - /// - /// Dictionary of object if and its subsequent closure table (a dictionary of hashes and min depth at which they are found). - /// - private Dictionary> RefMinDepthTracker { get; set; } - - public int TotalProcessedCount; - - #endregion - - #region Write Json - - // Keeps track of the actual tree structure of the objects being serialised. - // These tree references will thereafter be stored in the __tree prop. - private void TrackReferenceInTree(string refId) - { - // Help with creating closure table entries. - for (int i = 0; i < Lineage.Count; i++) - { - var parent = Lineage[i]; - - if (!RefMinDepthTracker.ContainsKey(parent)) - { - RefMinDepthTracker[parent] = new Dictionary(); - } - - if (!RefMinDepthTracker[parent].ContainsKey(refId)) - { - RefMinDepthTracker[parent][refId] = Lineage.Count - i; - } - else if (RefMinDepthTracker[parent][refId] > Lineage.Count - i) - { - RefMinDepthTracker[parent][refId] = Lineage.Count - i; - } - } - } - - private bool FirstEntry = true, - FirstEntryWasListOrDict; - - // While this function looks complicated, it's actually quite smooth: - // The important things to remember is that serialization goes depth first: - // The first object to get fully serialised is the first nested one, with - // the parent object being last. - public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) - { - writer.Formatting = serializer.Formatting; - if (CancellationToken.IsCancellationRequested) - { - return; // Check for cancellation - } - - ///////////////////////////////////// - // Path one: nulls - ///////////////////////////////////// - - if (value == null) - { - return; - } - - ///////////////////////////////////// - // Path two: primitives (string, bool, int, etc) - ///////////////////////////////////// - - if (value.GetType().IsPrimitive || value is string) - { - FirstEntry = false; - writer.WriteValue(value); - //var t = JToken.FromObject(value); // bypasses this converter as we do not pass in the serializer - //t.WriteTo(writer); - return; - } - - ///////////////////////////////////// - // Path three: Bases - ///////////////////////////////////// - - if (value is Base && !(value is ObjectReference)) - { - if (CancellationToken.IsCancellationRequested) - { - return; // Check for cancellation - } - - var obj = value as Base; - - FirstEntry = false; - //TotalProcessedCount++; - - // Append to lineage tracker - Lineage.Add(Guid.NewGuid().ToString()); - - var jo = new JObject(); - var propertyNames = obj.GetDynamicMemberNames(); - - var contract = (JsonDynamicContract)serializer.ContractResolver.ResolveContract(value.GetType()); - - // Iterate through the object's properties, one by one, checking for ignored ones - foreach (var prop in propertyNames) - { - if (CancellationToken.IsCancellationRequested) - { - return; // Check for cancellation - } - // Ignore properties starting with a double underscore. - if (prop.StartsWith("__")) - { - continue; - } - - if (prop == "id") - { - continue; - } - - var property = contract.Properties.GetClosestMatchProperty(prop); - - // Ignore properties decorated with [JsonIgnore]. - if (property != null && property.Ignored) - { - continue; - } - - // Ignore nulls - object propValue = obj[prop]; - if (propValue == null) - { - continue; - } - - // Check if this property is marked for detachment: either by the presence of "@" at the beginning of the name, or by the presence of a DetachProperty attribute on a typed property. - if (property != null) - { - var detachableAttributes = property.AttributeProvider.GetAttributes(typeof(DetachProperty), true); - if (detachableAttributes.Count > 0) - { - DetachLineage.Add(((DetachProperty)detachableAttributes[0]).Detachable); - } - else - { - DetachLineage.Add(false); - } - - var chunkableAttributes = property.AttributeProvider.GetAttributes(typeof(Chunkable), true); - if (chunkableAttributes.Count > 0) - { - //DetachLineage.Add(true); // NOOPE - serializer.Context = new StreamingContext(StreamingContextStates.Other, chunkableAttributes[0]); - } - else - { - //DetachLineage.Add(false); - serializer.Context = new StreamingContext(); - } - } - else if (prop.StartsWith("@")) // Convention check for dynamically added properties. - { - DetachLineage.Add(true); - - var chunkSyntax = Constants.ChunkPropertyNameRegex; - - if (chunkSyntax.IsMatch(prop)) - { - var match = chunkSyntax.Match(prop); - _ = int.TryParse(match.Groups[match.Groups.Count - 1].Value, out int chunkSize); - serializer.Context = new StreamingContext( - StreamingContextStates.Other, - chunkSize > 0 ? new Chunkable(chunkSize) : new Chunkable() - ); - } - else - { - serializer.Context = new StreamingContext(); - } - } - else - { - DetachLineage.Add(false); - } - - // Set and store a reference, if it is marked as detachable and the transport is not null. - if ( - WriteTransports != null - && WriteTransports.Count != 0 - && propValue is Base - && DetachLineage[DetachLineage.Count - 1] - ) - { - var what = JToken.FromObject(propValue, serializer); // Trigger next. - - if (CancellationToken.IsCancellationRequested) - { - return; // Check for cancellation - } - - if (what == null) - { - return; // HACK: Prevent nulls from borking our serialization on nested schema object refs. (i.e. Line has @SchemaObject, that has ref to line) - } - - var refHash = ((JObject)what).GetValue("id").ToString(); - - var reference = new ObjectReference { referencedId = refHash }; - TrackReferenceInTree(refHash); - jo.Add(prop, JToken.FromObject(reference)); - } - else - { - jo.Add(prop, JToken.FromObject(propValue, serializer)); // Default route - } - - // Pop detach lineage. If you don't get this, remember this thing moves ONLY FORWARD, DEPTH FIRST - DetachLineage.RemoveAt(DetachLineage.Count - 1); - // Refresh the streaming context to remove chunking flag - serializer.Context = new StreamingContext(); - } - - // Check if we actually have any transports present that would warrant a - if ( - WriteTransports != null - && WriteTransports.Count != 0 - && RefMinDepthTracker.ContainsKey(Lineage[Lineage.Count - 1]) - ) - { - jo.Add("__closure", JToken.FromObject(RefMinDepthTracker[Lineage[Lineage.Count - 1]])); - } - - var hash = Utilities.HashString(jo.ToString()); - if (!jo.ContainsKey("id")) - { - jo.Add("id", JToken.FromObject(hash)); - } - - jo.WriteTo(writer); - - if ( - (DetachLineage.Count == 0 || DetachLineage[DetachLineage.Count - 1]) - && WriteTransports != null - && WriteTransports.Count != 0 - ) - { - var objString = jo.ToString(writer.Formatting); - var objId = jo["id"].Value(); - - OnProgressAction?.Invoke("S", 1); - - foreach (var transport in WriteTransports) - { - if (CancellationToken.IsCancellationRequested) - { - continue; // Check for cancellation - } - - transport.SaveObject(objId, objString); - } - } - - // Pop lineage tracker - Lineage.RemoveAt(Lineage.Count - 1); - return; - } - - ///////////////////////////////////// - // Path four: lists/arrays & dicts - ///////////////////////////////////// - - if (CancellationToken.IsCancellationRequested) - { - return; // Check for cancellation - } - - var type = value.GetType(); - - // TODO: List handling and dictionary serialisation handling can be sped up significantly if we first check by their inner type. - // This handles a broader case in which we are, essentially, checking only for object[] or List / Dictionary cases. - // A much faster approach is to check for List, where primitive = string, number, etc. and directly serialize it in full. - // Same goes for dictionaries. - if ( - typeof(IEnumerable).IsAssignableFrom(type) - && !typeof(IDictionary).IsAssignableFrom(type) - && type != typeof(string) - ) - { - if (TotalProcessedCount == 0 && FirstEntry) - { - FirstEntry = false; - FirstEntryWasListOrDict = true; - TotalProcessedCount += 1; - DetachLineage.Add(WriteTransports != null && WriteTransports.Count != 0 ? true : false); - } - - JArray arr = new(); - - // Chunking large lists into manageable parts. - if (DetachLineage[DetachLineage.Count - 1] && serializer.Context.Context is Chunkable chunkInfo) - { - var maxCount = chunkInfo.MaxObjCountPerChunk; - var i = 0; - var chunkList = new List(); - var currChunk = new DataChunk(); - - foreach (var arrValue in (IEnumerable)value) - { - if (i == maxCount) - { - if (currChunk.data.Count != 0) - { - chunkList.Add(currChunk); - } - - currChunk = new DataChunk(); - i = 0; - } - currChunk.data.Add(arrValue); - i++; - } - - if (currChunk.data.Count != 0) - { - chunkList.Add(currChunk); - } - - value = chunkList; - } - - foreach (var arrValue in (IEnumerable)value) - { - if (CancellationToken.IsCancellationRequested) - { - return; // Check for cancellation - } - - if (arrValue == null) - { - continue; - } - - if ( - WriteTransports != null - && WriteTransports.Count != 0 - && arrValue is Base - && DetachLineage[DetachLineage.Count - 1] - ) - { - var what = JToken.FromObject(arrValue, serializer); // Trigger next - - var refHash = ((JObject)what).GetValue("id").ToString(); - - var reference = new ObjectReference { referencedId = refHash }; - TrackReferenceInTree(refHash); - arr.Add(JToken.FromObject(reference)); - } - else - { - arr.Add(JToken.FromObject(arrValue, serializer)); // Default route - } - } - - if (CancellationToken.IsCancellationRequested) - { - return; // Check for cancellation - } - - arr.WriteTo(writer); - - if (DetachLineage.Count == 1 && FirstEntryWasListOrDict) // are we in a list entry point case? - { - DetachLineage.RemoveAt(0); - } - - return; - } - - if (CancellationToken.IsCancellationRequested) - { - return; // Check for cancellation - } - - if (typeof(IDictionary).IsAssignableFrom(type)) - { - if (TotalProcessedCount == 0 && FirstEntry) - { - FirstEntry = false; - FirstEntryWasListOrDict = true; - TotalProcessedCount += 1; - DetachLineage.Add(WriteTransports != null && WriteTransports.Count != 0 ? true : false); - } - var dict = value as IDictionary; - var dictJo = new JObject(); - foreach (DictionaryEntry kvp in dict) - { - if (CancellationToken.IsCancellationRequested) - { - return; // Check for cancellation - } - - if (kvp.Value == null) - { - continue; - } - - JToken jToken; - if ( - WriteTransports != null - && WriteTransports.Count != 0 - && kvp.Value is Base - && DetachLineage[DetachLineage.Count - 1] - ) - { - var what = JToken.FromObject(kvp.Value, serializer); // Trigger next - var refHash = ((JObject)what).GetValue("id").ToString(); - - var reference = new ObjectReference { referencedId = refHash }; - TrackReferenceInTree(refHash); - jToken = JToken.FromObject(reference); - } - else - { - jToken = JToken.FromObject(kvp.Value, serializer); // Default route - } - dictJo.Add(kvp.Key.ToString(), jToken); - } - dictJo.WriteTo(writer); - - if (CancellationToken.IsCancellationRequested) - { - return; // Check for cancellation - } - - if (DetachLineage.Count == 1 && FirstEntryWasListOrDict) // are we in a dictionary entry point case? - { - DetachLineage.RemoveAt(0); - } - - return; - } - - ///////////////////////////////////// - // Path five: everything else (enums?) - ///////////////////////////////////// - - if (CancellationToken.IsCancellationRequested) - { - return; // Check for cancellation - } - - FirstEntry = false; - var lastCall = JToken.FromObject(value); // bypasses this converter as we do not pass in the serializer - lastCall.WriteTo(writer); - } - - #endregion -} -#pragma warning restore IDE0075, IDE1006, IDE0083, CA1051, CA1502, CA1854 diff --git a/src/Speckle.Objects/SpeckleSchemaInfo.cs b/src/Speckle.Objects/SpeckleSchemaInfo.cs new file mode 100644 index 0000000..b09699b --- /dev/null +++ b/src/Speckle.Objects/SpeckleSchemaInfo.cs @@ -0,0 +1,6 @@ +namespace Objects; + +public class SpeckleSchemaInfo +{ + public readonly Version Version = new(3, 0, 0); +} diff --git a/tests/Speckle.Core.Tests.Unit/Api/Operations/SendReceiveLocal.cs b/tests/Speckle.Core.Tests.Unit/Api/Operations/SendReceiveLocal.cs index 2bb9e62..86d2ba9 100644 --- a/tests/Speckle.Core.Tests.Unit/Api/Operations/SendReceiveLocal.cs +++ b/tests/Speckle.Core.Tests.Unit/Api/Operations/SendReceiveLocal.cs @@ -214,36 +214,7 @@ public async Task DownloadProgressReports() progress.NotNull(); Assert.That(progress.Keys, Has.Count.GreaterThanOrEqualTo(1)); } - - [Test(Description = "Should dispose of transports after a send or receive operation if so specified.")] - [Obsolete("Send overloads that perform disposal are deprecated")] - public async Task ShouldDisposeTransports() - { - var @base = new Base(); - @base["test"] = "the best"; - - var myLocalTransport = new SQLiteTransport(); - var id = await Core.Api.Operations.Send( - @base, - new List { myLocalTransport }, - false, - disposeTransports: true - ); - - // Send - Assert.ThrowsAsync( - async () => - await Core.Api.Operations.Send(@base, new List { myLocalTransport }, false, disposeTransports: true) - ); - - myLocalTransport = myLocalTransport.Clone() as SQLiteTransport; - _ = await Core.Api.Operations.Receive(id, null, myLocalTransport, disposeTransports: true); - - Assert.ThrowsAsync( - async () => await Core.Api.Operations.Receive(id, null, myLocalTransport) - ); - } - + [Test(Description = "Should not dispose of transports if so specified.")] public async Task ShouldNotDisposeTransports() { @@ -258,30 +229,6 @@ public async Task ShouldNotDisposeTransports() await Core.Api.Operations.Receive(id, null, myLocalTransport); } - //[Test] - //public async Task DiskTransportTest() - //{ - // var myObject = new Base(); - // myObject["@items"] = new List(); - // myObject["test"] = "random"; - - // var rand = new Random(); - - // for (int i = 0; i < 100; i++) - // { - // ((List)myObject["@items"]).Add(new Point(i, i, i) { applicationId = i + "-___/---" }); - // } - - // var dt = new Speckle.Core.Transports.Speckle.Speckle.Core.Transports(); - // var id = await Operations.Send(myObject, new List() { dt }, false); - - // Assert.IsNotNull(id); - - // var rebase = await Operations.Receive(id, dt); - - // Assert.AreEqual(rebase.GetId(true), id); - //} - public void Dispose() { _sut.Dispose(); From f36286ac80931a90ae7f34f6e5ac0fec5d2bfe2b Mon Sep 17 00:00:00 2001 From: Ian Hawley Date: Thu, 27 Jun 2024 20:22:07 +0100 Subject: [PATCH 02/18] added schema value <- before trying to remove core dependency> --- src/Speckle.Core/SpeckleObjectSchema.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 src/Speckle.Core/SpeckleObjectSchema.cs diff --git a/src/Speckle.Core/SpeckleObjectSchema.cs b/src/Speckle.Core/SpeckleObjectSchema.cs new file mode 100644 index 0000000..55901b5 --- /dev/null +++ b/src/Speckle.Core/SpeckleObjectSchema.cs @@ -0,0 +1,11 @@ +namespace Speckle.Core; + +// POC: core needs to know the version of the schema we have baked into the Objects DLL +// you'll notice this version isn't IN the objects DLL, that's because Objects is dependent upon core +// but core needs to know the current schema version we have loaded. Core should depend on objects and NOT +// the other way around. This needs some consideration but for POC of object versioning, this is where it lives. +public class SpeckleObjectSchema +{ + // POC: I'm not sure about using the Version object ATM, strings may be just as a straight-forward... + public static readonly Version Version = new Version(3, 0, 0); +} From fcd799d057175034ad1a0d8e58be664b6860cd50 Mon Sep 17 00:00:00 2001 From: Ian Hawley Date: Sun, 30 Jun 2024 12:51:43 +0100 Subject: [PATCH 03/18] trying to replace kit type system --- .../Serialisation/AbstractTypeCache.cs | 108 +++++ .../Serialisation/BaseObjectDeserializerV2.cs | 8 +- .../Serialisation/ISpeckleDeserializer.cs | 39 ++ .../BaseObjectSerializationUtilities._cs_ | 152 +++++++ .../BaseObjectSerializationUtilities.cs | 370 ------------------ .../BaseObjectSerializerV2TypeManager.cs | 146 +++++++ src/Speckle.Core/Speckle.Core.csproj | 4 + src/Speckle.Core/SpeckleObjectSchema.cs | 9 +- src/Speckle.Objects/ObjectsKit.cs | 13 +- src/Speckle.Objects/SpeckleBaseTypeCache.cs | 17 + 10 files changed, 483 insertions(+), 383 deletions(-) create mode 100644 src/Speckle.Core/Serialisation/AbstractTypeCache.cs create mode 100644 src/Speckle.Core/Serialisation/ISpeckleDeserializer.cs create mode 100644 src/Speckle.Core/Serialisation/SerializationUtilities/BaseObjectSerializationUtilities._cs_ delete mode 100644 src/Speckle.Core/Serialisation/SerializationUtilities/BaseObjectSerializationUtilities.cs create mode 100644 src/Speckle.Core/Serialisation/SerializationUtilities/BaseObjectSerializerV2TypeManager.cs create mode 100644 src/Speckle.Objects/SpeckleBaseTypeCache.cs diff --git a/src/Speckle.Core/Serialisation/AbstractTypeCache.cs b/src/Speckle.Core/Serialisation/AbstractTypeCache.cs new file mode 100644 index 0000000..2c8c7a2 --- /dev/null +++ b/src/Speckle.Core/Serialisation/AbstractTypeCache.cs @@ -0,0 +1,108 @@ +using System.Collections.Immutable; +using System.Reflection; +using Speckle.Core.Common; +using Speckle.Core.Models; + +namespace Speckle.Core.Serialisation; + +public class CachedType +{ + public string Key { get; private set; } + public Type Type { get; private set; } + public ImmutableDictionary Props { get; private set; } + + public CachedType(string key, Type type, Dictionary props) + { + Key = key; + Type = type; + Props = props.ToImmutableDictionary(); + } +} + +public abstract class AbstractTypeCache +{ + private readonly Assembly _assemblyToCache; + private readonly Type _baseType; + private readonly Dictionary _cachedTypes = new(); + + private CachedType? _defaultCachedType = null; + + protected AbstractTypeCache(Assembly assemblyToCache, Type baseType) + { + _assemblyToCache = assemblyToCache; + _baseType = baseType; + } + + public void EnsureCacheIsBuilt() + { + var foundTypes = _assemblyToCache.DefinedTypes.Where(x => x.IsSubclassOf(_baseType)); + + foreach (var type in foundTypes) + { + _cachedTypes.Add(type.FullName, new CachedType(type.FullName.NotNull(), type, GetPropertyInfo(type))); + } + + _defaultCachedType = new CachedType(_baseType.FullName.NotNull(), _baseType, GetPropertyInfo(_baseType)); + } + + public CachedType GetType(string speckleType) + { + int length = speckleType.Length; + int end = length - 1; + int start; + + // _defaultCachedType should be created by now otherwise we should explode + CachedType cachedType = _defaultCachedType.NotNull(); + + do + { + string typeName; + + start = speckleType.LastIndexOf(':', 0, end); + if (start < 0) + { + // we didn't find a : + typeName = speckleType.Substring(0, end); + } + else + { + typeName = speckleType.Substring(start + 1, end - start); + end = start - 1; + } + + if (_cachedTypes.TryGetValue(typeName, out cachedType)) + { + return cachedType; + } + + var lastPeriod = typeName.LastIndexOf('.'); + typeName = speckleType.Insert(lastPeriod, "Deprecated."); + if (_cachedTypes.TryGetValue(typeName, out cachedType)) + { + return cachedType; + } + } while (start >= 0); + + // why the hell is this moaning about it being null? + return cachedType; + } + + private Dictionary GetPropertyInfo(Type type) + { + Dictionary propertyMap = new(); + + PropertyInfo[] properties = type.GetProperties(); + foreach (PropertyInfo prop in properties) + { + propertyMap[prop.Name.ToLower()] = prop; + } + + return propertyMap; + } +} + +public class SpeckleBaseTypeCache : AbstractTypeCache +{ + public SpeckleBaseTypeCache() + : base(Assembly.GetExecutingAssembly(), typeof(Base)) { } +} diff --git a/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs b/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs index 89e84b3..73880cc 100644 --- a/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs +++ b/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs @@ -15,7 +15,7 @@ namespace Speckle.Core.Serialisation; -public sealed class BaseObjectDeserializerV2 +public sealed class BaseObjectDeserializerV2 : ISpeckleDeserializer { private bool _isBusy; private readonly object _callbackLock = new(); @@ -399,10 +399,4 @@ private Base Dict2Base(Dictionary dictObj) return baseObj; } - - [Obsolete("Use nameof(Base.speckle_type)")] - public string TypeDiscriminator => TYPE_DISCRIMINATOR; - - [Obsolete("OnErrorAction unused, deserializer will throw exceptions instead")] - public Action? OnErrorAction { get; set; } } diff --git a/src/Speckle.Core/Serialisation/ISpeckleDeserializer.cs b/src/Speckle.Core/Serialisation/ISpeckleDeserializer.cs new file mode 100644 index 0000000..15ff888 --- /dev/null +++ b/src/Speckle.Core/Serialisation/ISpeckleDeserializer.cs @@ -0,0 +1,39 @@ +using Speckle.Core.Models; +using Speckle.Core.Transports; +using Speckle.Newtonsoft.Json; +using Speckle.Newtonsoft.Json.Linq; + +namespace Speckle.Core.Serialisation; + +public interface ISpeckleDeserializer + where RootType : class +{ + CancellationToken CancellationToken { get; set; } + + /// + /// The sync transport. This transport will be used synchronously. + /// + ITransport ReadTransport { get; set; } + + Action? OnProgressAction { get; set; } + string? BlobStorageFolder { get; set; } + TimeSpan Elapsed { get; } + int WorkerThreadCount { get; set; } + + /// The JSON string of the object to be deserialized + /// A typed object deserialized from the + /// Thrown when + /// was null + /// cannot be deserialised to type + // /// did not contain the required json objects (closures) + RootType Deserialize(string rootObjectJson); + + /// + /// The deserialized object + /// was null + /// was not valid JSON + /// Failed to deserialize to the target type + object? DeserializeTransportObject(string objectJson); + + object? ConvertJsonElement(JToken doc); +} diff --git a/src/Speckle.Core/Serialisation/SerializationUtilities/BaseObjectSerializationUtilities._cs_ b/src/Speckle.Core/Serialisation/SerializationUtilities/BaseObjectSerializationUtilities._cs_ new file mode 100644 index 0000000..c699871 --- /dev/null +++ b/src/Speckle.Core/Serialisation/SerializationUtilities/BaseObjectSerializationUtilities._cs_ @@ -0,0 +1,152 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Runtime.Serialization; +using System.Threading; +using Speckle.Core.Kits; +using Speckle.Core.Logging; +using Speckle.Core.Models; +using Speckle.Newtonsoft.Json; +using Speckle.Newtonsoft.Json.Linq; +using Speckle.Newtonsoft.Json.Serialization; + +namespace Speckle.Core.Serialisation.SerializationUtilities; + +internal static class BaseObjectSerializationUtilities +{ + #region Getting Types + + private static Dictionary s_cachedTypes = new(); + + private static readonly Dictionary> s_typeProperties = new(); + + private static readonly Dictionary> s_onDeserializedCallbacks = new(); + + internal static Type GetType(string objFullType) + { + lock (s_cachedTypes) + { + if (s_cachedTypes.TryGetValue(objFullType, out Type? type1)) + { + return type1; + } + + var type = GetAtomicType(objFullType); + s_cachedTypes[objFullType] = type; + return type; + } + } + + internal static Type GetAtomicType(string objFullType) + { + var objectTypes = objFullType.Split(':').Reverse(); + foreach (var typeName in objectTypes) + { + //TODO: rather than getting the type from the first loaded kit that has it, maybe + //we get it from a specific Kit + var type = KitManager.Types.FirstOrDefault(tp => tp.FullName == typeName); + if (type != null) + { + return type; + } + + //To allow for backwards compatibility saving deserialization target types. + //We also check a ".Deprecated" prefixed namespace + string deprecatedTypeName = GetDeprecatedTypeName(typeName); + + var deprecatedType = KitManager.Types.FirstOrDefault(tp => tp.FullName == deprecatedTypeName); + if (deprecatedType != null) + { + return deprecatedType; + } + } + + return typeof(Base); + } + + internal static string GetDeprecatedTypeName(string typeName, string deprecatedSubstring = "Deprecated.") + { + int lastDotIndex = typeName.LastIndexOf('.'); + return typeName.Insert(lastDotIndex + 1, deprecatedSubstring); + } + + internal static Dictionary GetTypeProperties(string objFullType) + { + lock (s_typeProperties) + { + if (s_typeProperties.TryGetValue(objFullType, out Dictionary? value)) + { + return value; + } + + Dictionary ret = new(); + Type type = GetType(objFullType); + PropertyInfo[] properties = type.GetProperties(); + foreach (PropertyInfo prop in properties) + { + ret[prop.Name.ToLower()] = prop; + } + + value = ret; + s_typeProperties[objFullType] = value; + return value; + } + } + + internal static List GetOnDeserializedCallbacks(string objFullType) + { + // return new List(); + lock (s_onDeserializedCallbacks) + { + // System.Runtime.Serialization.Ca + if (s_onDeserializedCallbacks.TryGetValue(objFullType, out List? value)) + { + return value; + } + + List ret = new(); + Type type = GetType(objFullType); + MethodInfo[] methods = type.GetMethods(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + foreach (MethodInfo method in methods) + { + List onDeserializedAttributes = method + .GetCustomAttributes(true) + .ToList(); + if (onDeserializedAttributes.Count > 0) + { + ret.Add(method); + } + } + + value = ret; + s_onDeserializedCallbacks[objFullType] = value; + return value; + } + } + + internal static Type GetSystemOrSpeckleType(string typeName) + { + var systemType = Type.GetType(typeName); + if (systemType != null) + { + return systemType; + } + + return GetAtomicType(typeName); + } + + /// + /// Flushes kit's (discriminator, type) cache. Useful if you're dynamically loading more kits at runtime, that provide better coverage of what you're deserialising, and it's now somehow poisoned because the higher level types were not originally available. + /// + public static void FlushCachedTypes() + { + lock (s_cachedTypes) + { + s_cachedTypes = new Dictionary(); + } + } + + #endregion +} diff --git a/src/Speckle.Core/Serialisation/SerializationUtilities/BaseObjectSerializationUtilities.cs b/src/Speckle.Core/Serialisation/SerializationUtilities/BaseObjectSerializationUtilities.cs deleted file mode 100644 index e8f00ca..0000000 --- a/src/Speckle.Core/Serialisation/SerializationUtilities/BaseObjectSerializationUtilities.cs +++ /dev/null @@ -1,370 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; -using System.Runtime.Serialization; -using System.Threading; -using Speckle.Core.Kits; -using Speckle.Core.Logging; -using Speckle.Core.Models; -using Speckle.Newtonsoft.Json; -using Speckle.Newtonsoft.Json.Linq; -using Speckle.Newtonsoft.Json.Serialization; - -namespace Speckle.Core.Serialisation.SerializationUtilities; - -internal static class BaseObjectSerializationUtilities -{ - #region Getting Types - - private static Dictionary s_cachedTypes = new(); - - private static readonly Dictionary> s_typeProperties = new(); - - private static readonly Dictionary> s_onDeserializedCallbacks = new(); - - internal static Type GetType(string objFullType) - { - lock (s_cachedTypes) - { - if (s_cachedTypes.TryGetValue(objFullType, out Type? type1)) - { - return type1; - } - - var type = GetAtomicType(objFullType); - s_cachedTypes[objFullType] = type; - return type; - } - } - - internal static Type GetAtomicType(string objFullType) - { - var objectTypes = objFullType.Split(':').Reverse(); - foreach (var typeName in objectTypes) - { - //TODO: rather than getting the type from the first loaded kit that has it, maybe - //we get it from a specific Kit - var type = KitManager.Types.FirstOrDefault(tp => tp.FullName == typeName); - if (type != null) - { - return type; - } - - //To allow for backwards compatibility saving deserialization target types. - //We also check a ".Deprecated" prefixed namespace - string deprecatedTypeName = GetDeprecatedTypeName(typeName); - - var deprecatedType = KitManager.Types.FirstOrDefault(tp => tp.FullName == deprecatedTypeName); - if (deprecatedType != null) - { - return deprecatedType; - } - } - - return typeof(Base); - } - - internal static string GetDeprecatedTypeName(string typeName, string deprecatedSubstring = "Deprecated.") - { - int lastDotIndex = typeName.LastIndexOf('.'); - return typeName.Insert(lastDotIndex + 1, deprecatedSubstring); - } - - internal static Dictionary GetTypeProperties(string objFullType) - { - lock (s_typeProperties) - { - if (s_typeProperties.TryGetValue(objFullType, out Dictionary? value)) - { - return value; - } - - Dictionary ret = new(); - Type type = GetType(objFullType); - PropertyInfo[] properties = type.GetProperties(); - foreach (PropertyInfo prop in properties) - { - ret[prop.Name.ToLower()] = prop; - } - - value = ret; - s_typeProperties[objFullType] = value; - return value; - } - } - - internal static List GetOnDeserializedCallbacks(string objFullType) - { - // return new List(); - lock (s_onDeserializedCallbacks) - { - // System.Runtime.Serialization.Ca - if (s_onDeserializedCallbacks.TryGetValue(objFullType, out List? value)) - { - return value; - } - - List ret = new(); - Type type = GetType(objFullType); - MethodInfo[] methods = type.GetMethods(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); - foreach (MethodInfo method in methods) - { - List onDeserializedAttributes = method - .GetCustomAttributes(true) - .ToList(); - if (onDeserializedAttributes.Count > 0) - { - ret.Add(method); - } - } - - value = ret; - s_onDeserializedCallbacks[objFullType] = value; - return value; - } - } - - internal static Type GetSystemOrSpeckleType(string typeName) - { - var systemType = Type.GetType(typeName); - if (systemType != null) - { - return systemType; - } - - return GetAtomicType(typeName); - } - - /// - /// Flushes kit's (discriminator, type) cache. Useful if you're dynamically loading more kits at runtime, that provide better coverage of what you're deserialising, and it's now somehow poisoned because the higher level types were not originally available. - /// - public static void FlushCachedTypes() - { - lock (s_cachedTypes) - { - s_cachedTypes = new Dictionary(); - } - } - - #endregion - - #region Obsolete -#pragma warning disable CS8602, CA1502 - - private static readonly Dictionary s_cachedAbstractTypes = new(); - - [Obsolete("Only Used by Serializer V1")] - internal static object? HandleAbstractOriginalValue(JToken jToken, string assemblyQualifiedName) - { - if (s_cachedAbstractTypes.TryGetValue(assemblyQualifiedName, out Type? type)) - { - return jToken.ToObject(type); - } - - var pieces = assemblyQualifiedName.Split(',').Select(s => s.Trim()).ToArray(); - - var myAssembly = AppDomain.CurrentDomain.GetAssemblies().FirstOrDefault(ass => ass.GetName().Name == pieces[1]); - if (myAssembly == null) - { - throw new SpeckleException("Could not load abstract object's assembly."); - } - - var myType = myAssembly.GetType(pieces[0]); - if (myType == null) - { - throw new SpeckleException("Could not load abstract object's assembly."); - } - - s_cachedAbstractTypes[assemblyQualifiedName] = myType; - - return jToken.ToObject(myType); - } - - [Obsolete("Only used by serializer v1")] - internal static object? HandleValue( - JToken? value, - JsonSerializer serializer, - CancellationToken cancellationToken, - JsonProperty? jsonProperty = null, - string typeDiscriminator = "speckle_type" - ) - { - cancellationToken.ThrowIfCancellationRequested(); - - if (jsonProperty is { PropertyType: null }) - { - throw new ArgumentException($"Expected {nameof(JsonProperty.PropertyType)} to be non-null", nameof(jsonProperty)); - } - - switch (value) - { - case JValue jValue when jsonProperty != null: - return jValue.ToObject(jsonProperty.PropertyType); - case JValue jValue: - return jValue.Value; - // Lists - case JArray array when jsonProperty != null && jsonProperty.PropertyType.GetConstructor(Type.EmptyTypes) != null: - { - var arr = Activator.CreateInstance(jsonProperty.PropertyType); - - var addMethod = arr.GetType().GetMethod(nameof(IList.Add))!; - var hasGenericType = jsonProperty.PropertyType.GenericTypeArguments.Length != 0; - - foreach (var val in array) - { - cancellationToken.ThrowIfCancellationRequested(); - - if (val == null) - { - continue; - } - - var item = HandleValue(val, serializer, cancellationToken); - - if (item is DataChunk chunk) - { - foreach (var dataItem in chunk.data) - { - if (hasGenericType && !jsonProperty.PropertyType.GenericTypeArguments[0].IsInterface) - { - if (jsonProperty.PropertyType.GenericTypeArguments[0].IsAssignableFrom(dataItem.GetType())) - { - addMethod.Invoke(arr, new[] { dataItem }); - } - else - { - addMethod.Invoke( - arr, - new[] { Convert.ChangeType(dataItem, jsonProperty.PropertyType.GenericTypeArguments[0]) } - ); - } - } - else - { - addMethod.Invoke(arr, new[] { dataItem }); - } - } - } - else if (hasGenericType && !jsonProperty.PropertyType.GenericTypeArguments[0].IsInterface) - { - if (jsonProperty.PropertyType.GenericTypeArguments[0].IsAssignableFrom(item.GetType())) - { - addMethod.Invoke(arr, new[] { item }); - } - else - { - addMethod.Invoke( - arr, - new[] { Convert.ChangeType(item, jsonProperty.PropertyType.GenericTypeArguments[0]) } - ); - } - } - else - { - addMethod.Invoke(arr, new[] { item }); - } - } - return arr; - } - case JArray array when jsonProperty != null: - { - var arr = (IList) - Activator.CreateInstance(typeof(List<>).MakeGenericType(jsonProperty.PropertyType.GetElementType())); - - foreach (var val in array) - { - cancellationToken.ThrowIfCancellationRequested(); - - if (val == null) - { - continue; - } - - var item = HandleValue(val, serializer, cancellationToken); - if (item is DataChunk chunk) - { - foreach (var dataItem in chunk.data) - { - if (!jsonProperty.PropertyType.GetElementType()!.IsInterface) - { - arr.Add(Convert.ChangeType(dataItem, jsonProperty.PropertyType.GetElementType()!)); - } - else - { - arr.Add(dataItem); - } - } - } - else - { - if (!jsonProperty.PropertyType.GetElementType()!.IsInterface) - { - arr.Add(Convert.ChangeType(item, jsonProperty.PropertyType.GetElementType()!)); - } - else - { - arr.Add(item); - } - } - } - var actualArr = Array.CreateInstance(jsonProperty.PropertyType.GetElementType()!, arr.Count); - arr.CopyTo(actualArr, 0); - return actualArr; - } - case JArray array: - { - var arr = new List(); - foreach (var val in array) - { - cancellationToken.ThrowIfCancellationRequested(); - - if (val == null) - { - continue; - } - - var item = HandleValue(val, serializer, cancellationToken); - - if (item is DataChunk chunk) - { - arr.AddRange(chunk.data); - } - else - { - arr.Add(item); - } - } - return arr; - } - case JObject jObject when jObject.Property(typeDiscriminator) != null: - return jObject.ToObject(serializer); - case JObject jObject: - { - var dict = - jsonProperty != null - ? Activator.CreateInstance(jsonProperty.PropertyType) as IDictionary - : new Dictionary(); - foreach (var prop in jObject) - { - cancellationToken.ThrowIfCancellationRequested(); - - object key = prop.Key; - if (jsonProperty != null) - { - key = Convert.ChangeType(prop.Key, jsonProperty.PropertyType.GetGenericArguments()[0]); - } - - dict[key] = HandleValue(prop.Value, serializer, cancellationToken); - } - return dict; - } - default: - return null; - } - } -#pragma warning restore CS8602, CA1502 // Dereference of a possibly null reference. - - #endregion -} diff --git a/src/Speckle.Core/Serialisation/SerializationUtilities/BaseObjectSerializerV2TypeManager.cs b/src/Speckle.Core/Serialisation/SerializationUtilities/BaseObjectSerializerV2TypeManager.cs new file mode 100644 index 0000000..bbf2acf --- /dev/null +++ b/src/Speckle.Core/Serialisation/SerializationUtilities/BaseObjectSerializerV2TypeManager.cs @@ -0,0 +1,146 @@ +using System.Reflection; +using System.Runtime.Serialization; +using Speckle.Core.Kits; +using Speckle.Core.Models; + +namespace Speckle.Core.Serialisation.SerializationUtilities; + +internal static class BaseObjectSerializationUtilities +{ + #region Getting Types + + private static Dictionary s_cachedTypes = new(); + + private static readonly Dictionary> s_typeProperties = new(); + + private static readonly Dictionary> s_onDeserializedCallbacks = new(); + + internal static Type GetType(string objFullType) + { + lock (s_cachedTypes) + { + SpeckleObjectSchema.TypeCache?.EnsureCacheIsBuilt(); + var typeCache = SpeckleObjectSchema.TypeCache?.GetType(objFullType); + + if (s_cachedTypes.TryGetValue(objFullType, out Type? type1)) + { + return type1; + } + + var type = GetAtomicType(objFullType); + s_cachedTypes[objFullType] = type; + return type; + } + } + + internal static Type GetAtomicType(string objFullType) + { + var objectTypes = objFullType.Split(':').Reverse(); + foreach (var typeName in objectTypes) + { + //TODO: rather than getting the type from the first loaded kit that has it, maybe + //we get it from a specific Kit + var type = KitManager.Types.FirstOrDefault(tp => tp.FullName == typeName); + if (type != null) + { + return type; + } + + //To allow for backwards compatibility saving deserialization target types. + //We also check a ".Deprecated" prefixed namespace + string deprecatedTypeName = GetDeprecatedTypeName(typeName); + + var deprecatedType = KitManager.Types.FirstOrDefault(tp => tp.FullName == deprecatedTypeName); + if (deprecatedType != null) + { + return deprecatedType; + } + } + + return typeof(Base); + } + + internal static string GetDeprecatedTypeName(string typeName, string deprecatedSubstring = "Deprecated.") + { + int lastDotIndex = typeName.LastIndexOf('.'); + return typeName.Insert(lastDotIndex + 1, deprecatedSubstring); + } + + internal static Dictionary GetTypeProperties(string objFullType) + { + lock (s_typeProperties) + { + if (s_typeProperties.TryGetValue(objFullType, out Dictionary? value)) + { + return value; + } + + Dictionary ret = new(); + Type type = GetType(objFullType); + PropertyInfo[] properties = type.GetProperties(); + foreach (PropertyInfo prop in properties) + { + ret[prop.Name.ToLower()] = prop; + } + + value = ret; + s_typeProperties[objFullType] = value; + return value; + } + } + + internal static List GetOnDeserializedCallbacks(string objFullType) + { + // return new List(); + lock (s_onDeserializedCallbacks) + { + // System.Runtime.Serialization.Ca + if (s_onDeserializedCallbacks.TryGetValue(objFullType, out List? value)) + { + return value; + } + + List ret = new(); + Type type = GetType(objFullType); + MethodInfo[] methods = type.GetMethods(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + foreach (MethodInfo method in methods) + { + List onDeserializedAttributes = method + .GetCustomAttributes(true) + .ToList(); + if (onDeserializedAttributes.Count > 0) + { + ret.Add(method); + } + } + + value = ret; + s_onDeserializedCallbacks[objFullType] = value; + return value; + } + } + + internal static Type GetSystemOrSpeckleType(string typeName) + { + var systemType = Type.GetType(typeName); + if (systemType != null) + { + return systemType; + } + + return GetAtomicType(typeName); + } + + /// + /// Flushes kit's (discriminator, type) cache. Useful if you're dynamically loading more kits at runtime, that provide better coverage of what you're deserialising, and it's now somehow poisoned because the higher level types were not originally available. + /// + public static void FlushCachedTypes() + { + lock (s_cachedTypes) + { + s_cachedTypes = new Dictionary(); + } + } + + #endregion +} diff --git a/src/Speckle.Core/Speckle.Core.csproj b/src/Speckle.Core/Speckle.Core.csproj index 0c499dc..2c9df1a 100644 --- a/src/Speckle.Core/Speckle.Core.csproj +++ b/src/Speckle.Core/Speckle.Core.csproj @@ -36,5 +36,9 @@ + + + + diff --git a/src/Speckle.Core/SpeckleObjectSchema.cs b/src/Speckle.Core/SpeckleObjectSchema.cs index 55901b5..3bfbce6 100644 --- a/src/Speckle.Core/SpeckleObjectSchema.cs +++ b/src/Speckle.Core/SpeckleObjectSchema.cs @@ -1,4 +1,6 @@ -namespace Speckle.Core; +using Speckle.Core.Serialisation; + +namespace Speckle.Core; // POC: core needs to know the version of the schema we have baked into the Objects DLL // you'll notice this version isn't IN the objects DLL, that's because Objects is dependent upon core @@ -6,6 +8,9 @@ // the other way around. This needs some consideration but for POC of object versioning, this is where it lives. public class SpeckleObjectSchema { - // POC: I'm not sure about using the Version object ATM, strings may be just as a straight-forward... + // POC: I'm not sure about using the Version object ATM, strings may be just as a straight-forward... public static readonly Version Version = new Version(3, 0, 0); + + // POC: hacky way to check typing + public static AbstractTypeCache? TypeCache = null; } diff --git a/src/Speckle.Objects/ObjectsKit.cs b/src/Speckle.Objects/ObjectsKit.cs index 7a0def9..3c3e68f 100644 --- a/src/Speckle.Objects/ObjectsKit.cs +++ b/src/Speckle.Objects/ObjectsKit.cs @@ -1,12 +1,9 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; using System.Reflection; using Speckle.Core.Helpers; using Speckle.Core.Kits; using Speckle.Core.Logging; using Speckle.Core.Models; +using Speckle.Objects; namespace Objects; @@ -21,6 +18,14 @@ public class ObjectsKit : ISpeckleKit private List? _converters; + public SpeckleBaseTypeCache Cache { get; set; } + + public ObjectsKit() + { + Cache = new SpeckleBaseTypeCache(); + Cache.EnsureCacheIsBuilt(); + } + /// /// Local installations store objects in C:\Users\USERNAME\AppData\Roaming\Speckle\Kits\Objects /// Admin/System-wide installations in C:\ProgramData\Speckle\Kits\Objects diff --git a/src/Speckle.Objects/SpeckleBaseTypeCache.cs b/src/Speckle.Objects/SpeckleBaseTypeCache.cs new file mode 100644 index 0000000..a0772a6 --- /dev/null +++ b/src/Speckle.Objects/SpeckleBaseTypeCache.cs @@ -0,0 +1,17 @@ +using System.Reflection; +using Speckle.Core; +using Speckle.Core.Models; +using Speckle.Core.Serialisation; + +namespace Speckle.Objects; + +public class SpeckleBaseTypeCache : AbstractTypeCache +{ + static SpeckleBaseTypeCache() + { + SpeckleObjectSchema.TypeCache = new SpeckleBaseTypeCache(); + } + + public SpeckleBaseTypeCache() + : base(Assembly.GetExecutingAssembly(), typeof(Base)) { } +} From 8c051b689f84ff2070f2c46b1697ca5d762d256d Mon Sep 17 00:00:00 2001 From: Ian Hawley Date: Sun, 30 Jun 2024 21:42:20 +0100 Subject: [PATCH 04/18] Suplanting kits Type discovery --- .../Serialisation/AbstractTypeCache.cs | 66 ++++++++++++------- .../Serialisation/BaseObjectDeserializerV2.cs | 33 ++++++++-- .../BaseObjectSerializerV2TypeManager.cs | 15 ++++- src/Speckle.Core/SpeckleObjectSchema.cs | 3 - src/Speckle.Objects/ObjectsKit.cs | 9 --- src/Speckle.Objects/SpeckleBaseTypeCache.cs | 17 ----- 6 files changed, 83 insertions(+), 60 deletions(-) delete mode 100644 src/Speckle.Objects/SpeckleBaseTypeCache.cs diff --git a/src/Speckle.Core/Serialisation/AbstractTypeCache.cs b/src/Speckle.Core/Serialisation/AbstractTypeCache.cs index 2c8c7a2..5cba4c8 100644 --- a/src/Speckle.Core/Serialisation/AbstractTypeCache.cs +++ b/src/Speckle.Core/Serialisation/AbstractTypeCache.cs @@ -5,13 +5,13 @@ namespace Speckle.Core.Serialisation; -public class CachedType +public class CachedTypeInfo { public string Key { get; private set; } public Type Type { get; private set; } public ImmutableDictionary Props { get; private set; } - public CachedType(string key, Type type, Dictionary props) + public CachedTypeInfo(string key, Type type, Dictionary props) { Key = key; Type = type; @@ -19,50 +19,74 @@ public CachedType(string key, Type type, Dictionary props) } } -public abstract class AbstractTypeCache +public class TypeCacheManager { - private readonly Assembly _assemblyToCache; private readonly Type _baseType; - private readonly Dictionary _cachedTypes = new(); + private readonly Dictionary _cachedTypes = new(); - private CachedType? _defaultCachedType = null; + public CachedTypeInfo? FallbackType { get; private set; } - protected AbstractTypeCache(Assembly assemblyToCache, Type baseType) + public TypeCacheManager(Type baseType) { - _assemblyToCache = assemblyToCache; _baseType = baseType; } public void EnsureCacheIsBuilt() { - var foundTypes = _assemblyToCache.DefinedTypes.Where(x => x.IsSubclassOf(_baseType)); - - foreach (var type in foundTypes) + // POC: need a way to pick our own objects and not the DUI2 objects plus this is a touch weak :pain + foreach ( + var assembly in AppDomain.CurrentDomain.GetAssemblies().Where(x => x.FullName.ToLower().Contains("objects")) + ) { - _cachedTypes.Add(type.FullName, new CachedType(type.FullName.NotNull(), type, GetPropertyInfo(type))); + try + { + var foundTypes = assembly.DefinedTypes.Where(x => x.IsSubclassOf(_baseType) && !x.IsAbstract); + + foreach (var type in foundTypes) + { + var typeName = type.FullName; + + // POC: I am not convinced we should have duplicates | plus the original Objects.Dll could be loaded at the same time + if (!_cachedTypes.ContainsKey(typeName)) + { + try + { + _cachedTypes.Add(typeName, new CachedTypeInfo(typeName.NotNull(), type, GetPropertyInfo(type))); + } + catch (TypeLoadException) + { + // POC: guard against loading things that cause explosions + } + } + } + } + catch (TypeLoadException) + { + // POC: guard against loading things that cause explosions + } } - _defaultCachedType = new CachedType(_baseType.FullName.NotNull(), _baseType, GetPropertyInfo(_baseType)); + FallbackType = new CachedTypeInfo(_baseType.FullName.NotNull(), _baseType, GetPropertyInfo(_baseType)); } - public CachedType GetType(string speckleType) + public CachedTypeInfo GetType(string speckleType) { int length = speckleType.Length; int end = length - 1; int start; // _defaultCachedType should be created by now otherwise we should explode - CachedType cachedType = _defaultCachedType.NotNull(); + CachedTypeInfo cachedType = FallbackType.NotNull(); do { string typeName; - start = speckleType.LastIndexOf(':', 0, end); + start = speckleType.LastIndexOf(':', end); if (start < 0) { // we didn't find a : - typeName = speckleType.Substring(0, end); + typeName = speckleType.Substring(0, 1 + end); } else { @@ -76,7 +100,7 @@ public CachedType GetType(string speckleType) } var lastPeriod = typeName.LastIndexOf('.'); - typeName = speckleType.Insert(lastPeriod, "Deprecated."); + typeName = typeName.Insert(1 + lastPeriod, "Deprecated."); if (_cachedTypes.TryGetValue(typeName, out cachedType)) { return cachedType; @@ -100,9 +124,3 @@ public CachedType GetType(string speckleType) return propertyMap; } } - -public class SpeckleBaseTypeCache : AbstractTypeCache -{ - public SpeckleBaseTypeCache() - : base(Assembly.GetExecutingAssembly(), typeof(Base)) { } -} diff --git a/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs b/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs index 73880cc..cb1f3db 100644 --- a/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs +++ b/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs @@ -45,6 +45,14 @@ public sealed class BaseObjectDeserializerV2 : ISpeckleDeserializer public static int DefaultNumberThreads => Math.Min(Environment.ProcessorCount, 6); //6 threads seems the sweet spot, see performance test project public int WorkerThreadCount { get; set; } = DefaultNumberThreads; + private readonly TypeCacheManager _typeCacheManager = new(typeof(Base)); + + // POC: inject the TypeCacheManager, and interface out + public BaseObjectDeserializerV2() + { + _typeCacheManager.EnsureCacheIsBuilt(); + } + /// The JSON string of the object to be deserialized /// A typed object deserialized from the /// Thrown when @@ -341,21 +349,36 @@ private List<(string, int)> GetClosures(string rootObjectJson) private Base Dict2Base(Dictionary dictObj) { string typeName = (string)dictObj[TYPE_DISCRIMINATOR]!; - Type type = BaseObjectSerializationUtilities.GetType(typeName); - Base baseObj = (Base)Activator.CreateInstance(type); + + CachedTypeInfo cachedTypeInfo = _typeCacheManager.GetType(typeName); + + var originalType = BaseObjectSerializationUtilities.GetType(typeName); + + // if (cachedTypeInfo.Type.FullName.StartsWith("Speckle.Connectors.Utils")) + // { + // cachedTypeInfo = _typeCacheManager.FallbackType.NotNull(); + // } + + Debug.WriteLine($"\t{cachedTypeInfo.Type.FullName} | {originalType.FullName}"); + Base baseObj = (Base)Activator.CreateInstance(cachedTypeInfo.Type); dictObj.Remove(TYPE_DISCRIMINATOR); dictObj.Remove("__closure"); - Dictionary staticProperties = BaseObjectSerializationUtilities.GetTypeProperties(typeName); + var props = cachedTypeInfo.Props; + //var props = BaseObjectSerializationUtilities.GetTypeProperties(typeName); + + //Dictionary staticProperties = BaseObjectSerializationUtilities.GetTypeProperties(typeName); + + List onDeserializedCallbacks = BaseObjectSerializationUtilities.GetOnDeserializedCallbacks(typeName); foreach (var entry in dictObj) { string lowerPropertyName = entry.Key.ToLower(); - if (staticProperties.TryGetValue(lowerPropertyName, out PropertyInfo? value) && value.CanWrite) + if (props.TryGetValue(lowerPropertyName, out PropertyInfo? value) && value.CanWrite) { - PropertyInfo property = staticProperties[lowerPropertyName]; + PropertyInfo property = props[lowerPropertyName]; if (entry.Value == null) { // Check for JsonProperty(NullValueHandling = NullValueHandling.Ignore) attribute diff --git a/src/Speckle.Core/Serialisation/SerializationUtilities/BaseObjectSerializerV2TypeManager.cs b/src/Speckle.Core/Serialisation/SerializationUtilities/BaseObjectSerializerV2TypeManager.cs index bbf2acf..b8d711e 100644 --- a/src/Speckle.Core/Serialisation/SerializationUtilities/BaseObjectSerializerV2TypeManager.cs +++ b/src/Speckle.Core/Serialisation/SerializationUtilities/BaseObjectSerializerV2TypeManager.cs @@ -15,12 +15,23 @@ internal static class BaseObjectSerializationUtilities private static readonly Dictionary> s_onDeserializedCallbacks = new(); + private static TypeCacheManager s_typeCache = new TypeCacheManager(typeof(Base)); + internal static Type GetType(string objFullType) { lock (s_cachedTypes) { - SpeckleObjectSchema.TypeCache?.EnsureCacheIsBuilt(); - var typeCache = SpeckleObjectSchema.TypeCache?.GetType(objFullType); + s_typeCache.EnsureCacheIsBuilt(); + var typeCache = s_typeCache.GetType(objFullType); + + var revitWall = s_typeCache.GetType("Objects.BuiltElements.Wall:Objects.BuiltElements.Revit.RevitWall"); + var wall = s_typeCache.GetType("Objects.BuiltElements.Wall:Objects.BuiltElements.Revit.NotARevitWall"); + var wall3deep = s_typeCache.GetType( + "Objects.BuiltElements.Wall:Objects.BuiltElements.Level2:Objects.BuiltElements.Revit.NotARevitWall" + ); + var wall2deep = s_typeCache.GetType( + "Objects.BuiltElements.NotAUsefulName:Objects.BuiltElements.Wall:Objects.BuiltElements.Revit.NotARevitWall" + ); if (s_cachedTypes.TryGetValue(objFullType, out Type? type1)) { diff --git a/src/Speckle.Core/SpeckleObjectSchema.cs b/src/Speckle.Core/SpeckleObjectSchema.cs index 3bfbce6..e3bcb4c 100644 --- a/src/Speckle.Core/SpeckleObjectSchema.cs +++ b/src/Speckle.Core/SpeckleObjectSchema.cs @@ -10,7 +10,4 @@ public class SpeckleObjectSchema { // POC: I'm not sure about using the Version object ATM, strings may be just as a straight-forward... public static readonly Version Version = new Version(3, 0, 0); - - // POC: hacky way to check typing - public static AbstractTypeCache? TypeCache = null; } diff --git a/src/Speckle.Objects/ObjectsKit.cs b/src/Speckle.Objects/ObjectsKit.cs index 3c3e68f..8962da1 100644 --- a/src/Speckle.Objects/ObjectsKit.cs +++ b/src/Speckle.Objects/ObjectsKit.cs @@ -3,7 +3,6 @@ using Speckle.Core.Kits; using Speckle.Core.Logging; using Speckle.Core.Models; -using Speckle.Objects; namespace Objects; @@ -18,14 +17,6 @@ public class ObjectsKit : ISpeckleKit private List? _converters; - public SpeckleBaseTypeCache Cache { get; set; } - - public ObjectsKit() - { - Cache = new SpeckleBaseTypeCache(); - Cache.EnsureCacheIsBuilt(); - } - /// /// Local installations store objects in C:\Users\USERNAME\AppData\Roaming\Speckle\Kits\Objects /// Admin/System-wide installations in C:\ProgramData\Speckle\Kits\Objects diff --git a/src/Speckle.Objects/SpeckleBaseTypeCache.cs b/src/Speckle.Objects/SpeckleBaseTypeCache.cs deleted file mode 100644 index a0772a6..0000000 --- a/src/Speckle.Objects/SpeckleBaseTypeCache.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System.Reflection; -using Speckle.Core; -using Speckle.Core.Models; -using Speckle.Core.Serialisation; - -namespace Speckle.Objects; - -public class SpeckleBaseTypeCache : AbstractTypeCache -{ - static SpeckleBaseTypeCache() - { - SpeckleObjectSchema.TypeCache = new SpeckleBaseTypeCache(); - } - - public SpeckleBaseTypeCache() - : base(Assembly.GetExecutingAssembly(), typeof(Base)) { } -} From daef155e3fa1ae3f34bc07468945e254271628ea Mon Sep 17 00:00:00 2001 From: Ian Hawley Date: Sun, 30 Jun 2024 23:35:49 +0100 Subject: [PATCH 05/18] Replaced the type information coming from kits with a new injectable type cache, which (if it works as expected) I think is a bit more efficient --- src/Speckle.Core/Api/Helpers.cs | 3 + .../Api/Operations/Operations.Receive.cs | 4 +- .../Api/Operations/Operations.Serialize.cs | 5 +- .../Serialisation/BaseObjectDeserializerV2.cs | 34 ++-- .../BaseObjectSerializationUtilities._cs_ | 152 ----------------- .../BaseObjectSerializerV2TypeManager.cs | 157 ------------------ .../{ => TypeCache}/AbstractTypeCache.cs | 73 +++++--- .../Serialisation/TypeCache/CachedTypeInfo.cs | 24 +++ .../Serialisation/TypeCache/ITypeCache.cs | 7 + src/Speckle.Objects/ObjectsKit.cs | 9 +- src/Speckle.Objects/ObjectsTypeCache.cs | 9 + 11 files changed, 118 insertions(+), 359 deletions(-) delete mode 100644 src/Speckle.Core/Serialisation/SerializationUtilities/BaseObjectSerializationUtilities._cs_ delete mode 100644 src/Speckle.Core/Serialisation/SerializationUtilities/BaseObjectSerializerV2TypeManager.cs rename src/Speckle.Core/Serialisation/{ => TypeCache}/AbstractTypeCache.cs (56%) create mode 100644 src/Speckle.Core/Serialisation/TypeCache/CachedTypeInfo.cs create mode 100644 src/Speckle.Core/Serialisation/TypeCache/ITypeCache.cs create mode 100644 src/Speckle.Objects/ObjectsTypeCache.cs diff --git a/src/Speckle.Core/Api/Helpers.cs b/src/Speckle.Core/Api/Helpers.cs index 313e1b8..e110119 100644 --- a/src/Speckle.Core/Api/Helpers.cs +++ b/src/Speckle.Core/Api/Helpers.cs @@ -15,6 +15,7 @@ using Speckle.Core.Kits; using Speckle.Core.Logging; using Speckle.Core.Models; +using Speckle.Core.Serialisation.TypeCache; using Speckle.Core.Transports; using Speckle.Newtonsoft.Json; @@ -35,6 +36,7 @@ public static class Helpers /// public static async Task Receive( string stream, + ITypeCache typeCache, Account account = null, Action> onProgressAction = null, Action onTotalChildrenCountKnown = null @@ -107,6 +109,7 @@ public static class Helpers var receiveRes = await Operations .Receive( objectId, + typeCache, transport, onProgressAction: onProgressAction, onTotalChildrenCountKnown: onTotalChildrenCountKnown diff --git a/src/Speckle.Core/Api/Operations/Operations.Receive.cs b/src/Speckle.Core/Api/Operations/Operations.Receive.cs index 2502b6d..0c597ee 100644 --- a/src/Speckle.Core/Api/Operations/Operations.Receive.cs +++ b/src/Speckle.Core/Api/Operations/Operations.Receive.cs @@ -9,6 +9,7 @@ using Speckle.Core.Logging; using Speckle.Core.Models; using Speckle.Core.Serialisation; +using Speckle.Core.Serialisation.TypeCache; using Speckle.Core.Transports; namespace Speckle.Core.Api; @@ -38,6 +39,7 @@ public static partial class Operations /// The requested Speckle Object public static async Task Receive( string objectId, + ITypeCache typeCache, ITransport? remoteTransport = null, ITransport? localTransport = null, Action>? onProgressAction = null, @@ -62,7 +64,7 @@ public static partial class Operations // Setup Serializer BaseObjectDeserializerV2 serializerV2 = - new() + new(typeCache) { ReadTransport = localTransport, OnProgressAction = internalProgressAction, diff --git a/src/Speckle.Core/Api/Operations/Operations.Serialize.cs b/src/Speckle.Core/Api/Operations/Operations.Serialize.cs index 6fa3d89..87a206f 100644 --- a/src/Speckle.Core/Api/Operations/Operations.Serialize.cs +++ b/src/Speckle.Core/Api/Operations/Operations.Serialize.cs @@ -1,6 +1,7 @@ using Speckle.Core.Logging; using Speckle.Core.Models; using Speckle.Core.Serialisation; +using Speckle.Core.Serialisation.TypeCache; using Speckle.Newtonsoft.Json; namespace Speckle.Core.Api; @@ -36,9 +37,9 @@ public static string Serialize(Base value, CancellationToken cancellationToken = /// was not valid JSON /// cannot be deserialised to type /// contains closure references (see Remarks) - public static Base Deserialize(string value, CancellationToken cancellationToken = default) + public static Base Deserialize(string value, ITypeCache typeCache, CancellationToken cancellationToken = default) { - var deserializer = new BaseObjectDeserializerV2 { CancellationToken = cancellationToken }; + var deserializer = new BaseObjectDeserializerV2(typeCache) { CancellationToken = cancellationToken }; return deserializer.Deserialize(value); } } diff --git a/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs b/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs index cb1f3db..90703aa 100644 --- a/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs +++ b/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs @@ -6,9 +6,11 @@ using System.Threading; using System.Threading.Tasks; using Speckle.Core.Common; +using Speckle.Core.Kits; using Speckle.Core.Logging; using Speckle.Core.Models; using Speckle.Core.Serialisation.SerializationUtilities; +using Speckle.Core.Serialisation.TypeCache; using Speckle.Core.Transports; using Speckle.Newtonsoft.Json; using Speckle.Newtonsoft.Json.Linq; @@ -45,12 +47,13 @@ public sealed class BaseObjectDeserializerV2 : ISpeckleDeserializer public static int DefaultNumberThreads => Math.Min(Environment.ProcessorCount, 6); //6 threads seems the sweet spot, see performance test project public int WorkerThreadCount { get; set; } = DefaultNumberThreads; - private readonly TypeCacheManager _typeCacheManager = new(typeof(Base)); + private readonly ITypeCache _typeCache; // POC: inject the TypeCacheManager, and interface out - public BaseObjectDeserializerV2() + public BaseObjectDeserializerV2(ITypeCache typeCache) { - _typeCacheManager.EnsureCacheIsBuilt(); + _typeCache = typeCache; + _typeCache.EnsureCacheIsBuilt(); } /// The JSON string of the object to be deserialized @@ -349,29 +352,22 @@ private List<(string, int)> GetClosures(string rootObjectJson) private Base Dict2Base(Dictionary dictObj) { string typeName = (string)dictObj[TYPE_DISCRIMINATOR]!; + + // TODO: get the version - CachedTypeInfo cachedTypeInfo = _typeCacheManager.GetType(typeName); + // we find the type + CachedTypeInfo cachedTypeInfo = _typeCache.GetType(typeName); - var originalType = BaseObjectSerializationUtilities.GetType(typeName); - - // if (cachedTypeInfo.Type.FullName.StartsWith("Speckle.Connectors.Utils")) - // { - // cachedTypeInfo = _typeCacheManager.FallbackType.NotNull(); - // } - - Debug.WriteLine($"\t{cachedTypeInfo.Type.FullName} | {originalType.FullName}"); - Base baseObj = (Base)Activator.CreateInstance(cachedTypeInfo.Type); + // let's figure out whether to deserialise into this type or a legacy type + + Base baseObj = (Base) Activator.CreateInstance(cachedTypeInfo.Type); dictObj.Remove(TYPE_DISCRIMINATOR); dictObj.Remove("__closure"); var props = cachedTypeInfo.Props; - //var props = BaseObjectSerializationUtilities.GetTypeProperties(typeName); - - //Dictionary staticProperties = BaseObjectSerializationUtilities.GetTypeProperties(typeName); - - List onDeserializedCallbacks = BaseObjectSerializationUtilities.GetOnDeserializedCallbacks(typeName); + var onDeserializedCallbacks = cachedTypeInfo.Callbacks; foreach (var entry in dictObj) { @@ -414,6 +410,8 @@ private Base Dict2Base(Dictionary dictObj) { bb.filePath = bb.GetLocalDestinationPath(BlobStorageFolder); } + + // TODO: perform any versioning we need to foreach (MethodInfo onDeserialized in onDeserializedCallbacks) { diff --git a/src/Speckle.Core/Serialisation/SerializationUtilities/BaseObjectSerializationUtilities._cs_ b/src/Speckle.Core/Serialisation/SerializationUtilities/BaseObjectSerializationUtilities._cs_ deleted file mode 100644 index c699871..0000000 --- a/src/Speckle.Core/Serialisation/SerializationUtilities/BaseObjectSerializationUtilities._cs_ +++ /dev/null @@ -1,152 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; -using System.Runtime.Serialization; -using System.Threading; -using Speckle.Core.Kits; -using Speckle.Core.Logging; -using Speckle.Core.Models; -using Speckle.Newtonsoft.Json; -using Speckle.Newtonsoft.Json.Linq; -using Speckle.Newtonsoft.Json.Serialization; - -namespace Speckle.Core.Serialisation.SerializationUtilities; - -internal static class BaseObjectSerializationUtilities -{ - #region Getting Types - - private static Dictionary s_cachedTypes = new(); - - private static readonly Dictionary> s_typeProperties = new(); - - private static readonly Dictionary> s_onDeserializedCallbacks = new(); - - internal static Type GetType(string objFullType) - { - lock (s_cachedTypes) - { - if (s_cachedTypes.TryGetValue(objFullType, out Type? type1)) - { - return type1; - } - - var type = GetAtomicType(objFullType); - s_cachedTypes[objFullType] = type; - return type; - } - } - - internal static Type GetAtomicType(string objFullType) - { - var objectTypes = objFullType.Split(':').Reverse(); - foreach (var typeName in objectTypes) - { - //TODO: rather than getting the type from the first loaded kit that has it, maybe - //we get it from a specific Kit - var type = KitManager.Types.FirstOrDefault(tp => tp.FullName == typeName); - if (type != null) - { - return type; - } - - //To allow for backwards compatibility saving deserialization target types. - //We also check a ".Deprecated" prefixed namespace - string deprecatedTypeName = GetDeprecatedTypeName(typeName); - - var deprecatedType = KitManager.Types.FirstOrDefault(tp => tp.FullName == deprecatedTypeName); - if (deprecatedType != null) - { - return deprecatedType; - } - } - - return typeof(Base); - } - - internal static string GetDeprecatedTypeName(string typeName, string deprecatedSubstring = "Deprecated.") - { - int lastDotIndex = typeName.LastIndexOf('.'); - return typeName.Insert(lastDotIndex + 1, deprecatedSubstring); - } - - internal static Dictionary GetTypeProperties(string objFullType) - { - lock (s_typeProperties) - { - if (s_typeProperties.TryGetValue(objFullType, out Dictionary? value)) - { - return value; - } - - Dictionary ret = new(); - Type type = GetType(objFullType); - PropertyInfo[] properties = type.GetProperties(); - foreach (PropertyInfo prop in properties) - { - ret[prop.Name.ToLower()] = prop; - } - - value = ret; - s_typeProperties[objFullType] = value; - return value; - } - } - - internal static List GetOnDeserializedCallbacks(string objFullType) - { - // return new List(); - lock (s_onDeserializedCallbacks) - { - // System.Runtime.Serialization.Ca - if (s_onDeserializedCallbacks.TryGetValue(objFullType, out List? value)) - { - return value; - } - - List ret = new(); - Type type = GetType(objFullType); - MethodInfo[] methods = type.GetMethods(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); - foreach (MethodInfo method in methods) - { - List onDeserializedAttributes = method - .GetCustomAttributes(true) - .ToList(); - if (onDeserializedAttributes.Count > 0) - { - ret.Add(method); - } - } - - value = ret; - s_onDeserializedCallbacks[objFullType] = value; - return value; - } - } - - internal static Type GetSystemOrSpeckleType(string typeName) - { - var systemType = Type.GetType(typeName); - if (systemType != null) - { - return systemType; - } - - return GetAtomicType(typeName); - } - - /// - /// Flushes kit's (discriminator, type) cache. Useful if you're dynamically loading more kits at runtime, that provide better coverage of what you're deserialising, and it's now somehow poisoned because the higher level types were not originally available. - /// - public static void FlushCachedTypes() - { - lock (s_cachedTypes) - { - s_cachedTypes = new Dictionary(); - } - } - - #endregion -} diff --git a/src/Speckle.Core/Serialisation/SerializationUtilities/BaseObjectSerializerV2TypeManager.cs b/src/Speckle.Core/Serialisation/SerializationUtilities/BaseObjectSerializerV2TypeManager.cs deleted file mode 100644 index b8d711e..0000000 --- a/src/Speckle.Core/Serialisation/SerializationUtilities/BaseObjectSerializerV2TypeManager.cs +++ /dev/null @@ -1,157 +0,0 @@ -using System.Reflection; -using System.Runtime.Serialization; -using Speckle.Core.Kits; -using Speckle.Core.Models; - -namespace Speckle.Core.Serialisation.SerializationUtilities; - -internal static class BaseObjectSerializationUtilities -{ - #region Getting Types - - private static Dictionary s_cachedTypes = new(); - - private static readonly Dictionary> s_typeProperties = new(); - - private static readonly Dictionary> s_onDeserializedCallbacks = new(); - - private static TypeCacheManager s_typeCache = new TypeCacheManager(typeof(Base)); - - internal static Type GetType(string objFullType) - { - lock (s_cachedTypes) - { - s_typeCache.EnsureCacheIsBuilt(); - var typeCache = s_typeCache.GetType(objFullType); - - var revitWall = s_typeCache.GetType("Objects.BuiltElements.Wall:Objects.BuiltElements.Revit.RevitWall"); - var wall = s_typeCache.GetType("Objects.BuiltElements.Wall:Objects.BuiltElements.Revit.NotARevitWall"); - var wall3deep = s_typeCache.GetType( - "Objects.BuiltElements.Wall:Objects.BuiltElements.Level2:Objects.BuiltElements.Revit.NotARevitWall" - ); - var wall2deep = s_typeCache.GetType( - "Objects.BuiltElements.NotAUsefulName:Objects.BuiltElements.Wall:Objects.BuiltElements.Revit.NotARevitWall" - ); - - if (s_cachedTypes.TryGetValue(objFullType, out Type? type1)) - { - return type1; - } - - var type = GetAtomicType(objFullType); - s_cachedTypes[objFullType] = type; - return type; - } - } - - internal static Type GetAtomicType(string objFullType) - { - var objectTypes = objFullType.Split(':').Reverse(); - foreach (var typeName in objectTypes) - { - //TODO: rather than getting the type from the first loaded kit that has it, maybe - //we get it from a specific Kit - var type = KitManager.Types.FirstOrDefault(tp => tp.FullName == typeName); - if (type != null) - { - return type; - } - - //To allow for backwards compatibility saving deserialization target types. - //We also check a ".Deprecated" prefixed namespace - string deprecatedTypeName = GetDeprecatedTypeName(typeName); - - var deprecatedType = KitManager.Types.FirstOrDefault(tp => tp.FullName == deprecatedTypeName); - if (deprecatedType != null) - { - return deprecatedType; - } - } - - return typeof(Base); - } - - internal static string GetDeprecatedTypeName(string typeName, string deprecatedSubstring = "Deprecated.") - { - int lastDotIndex = typeName.LastIndexOf('.'); - return typeName.Insert(lastDotIndex + 1, deprecatedSubstring); - } - - internal static Dictionary GetTypeProperties(string objFullType) - { - lock (s_typeProperties) - { - if (s_typeProperties.TryGetValue(objFullType, out Dictionary? value)) - { - return value; - } - - Dictionary ret = new(); - Type type = GetType(objFullType); - PropertyInfo[] properties = type.GetProperties(); - foreach (PropertyInfo prop in properties) - { - ret[prop.Name.ToLower()] = prop; - } - - value = ret; - s_typeProperties[objFullType] = value; - return value; - } - } - - internal static List GetOnDeserializedCallbacks(string objFullType) - { - // return new List(); - lock (s_onDeserializedCallbacks) - { - // System.Runtime.Serialization.Ca - if (s_onDeserializedCallbacks.TryGetValue(objFullType, out List? value)) - { - return value; - } - - List ret = new(); - Type type = GetType(objFullType); - MethodInfo[] methods = type.GetMethods(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); - foreach (MethodInfo method in methods) - { - List onDeserializedAttributes = method - .GetCustomAttributes(true) - .ToList(); - if (onDeserializedAttributes.Count > 0) - { - ret.Add(method); - } - } - - value = ret; - s_onDeserializedCallbacks[objFullType] = value; - return value; - } - } - - internal static Type GetSystemOrSpeckleType(string typeName) - { - var systemType = Type.GetType(typeName); - if (systemType != null) - { - return systemType; - } - - return GetAtomicType(typeName); - } - - /// - /// Flushes kit's (discriminator, type) cache. Useful if you're dynamically loading more kits at runtime, that provide better coverage of what you're deserialising, and it's now somehow poisoned because the higher level types were not originally available. - /// - public static void FlushCachedTypes() - { - lock (s_cachedTypes) - { - s_cachedTypes = new Dictionary(); - } - } - - #endregion -} diff --git a/src/Speckle.Core/Serialisation/AbstractTypeCache.cs b/src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs similarity index 56% rename from src/Speckle.Core/Serialisation/AbstractTypeCache.cs rename to src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs index 5cba4c8..979565d 100644 --- a/src/Speckle.Core/Serialisation/AbstractTypeCache.cs +++ b/src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs @@ -1,46 +1,36 @@ using System.Collections.Immutable; using System.Reflection; +using System.Runtime.Serialization; using Speckle.Core.Common; using Speckle.Core.Models; -namespace Speckle.Core.Serialisation; +namespace Speckle.Core.Serialisation.TypeCache; -public class CachedTypeInfo -{ - public string Key { get; private set; } - public Type Type { get; private set; } - public ImmutableDictionary Props { get; private set; } - - public CachedTypeInfo(string key, Type type, Dictionary props) - { - Key = key; - Type = type; - Props = props.ToImmutableDictionary(); - } -} - -public class TypeCacheManager +public abstract class AbstractTypeCache : ITypeCache { + private readonly IList _assemblies; private readonly Type _baseType; private readonly Dictionary _cachedTypes = new(); public CachedTypeInfo? FallbackType { get; private set; } - public TypeCacheManager(Type baseType) + protected AbstractTypeCache(IEnumerable assemblies, Type baseType) { + _assemblies = assemblies.ToList(); _baseType = baseType; + + // POC: manually including core, not sure of the wisdom of this... + _assemblies.Add(typeof(AbstractTypeCache).Assembly); } public void EnsureCacheIsBuilt() { // POC: need a way to pick our own objects and not the DUI2 objects plus this is a touch weak :pain - foreach ( - var assembly in AppDomain.CurrentDomain.GetAssemblies().Where(x => x.FullName.ToLower().Contains("objects")) - ) + foreach (var assembly in _assemblies) { try { - var foundTypes = assembly.DefinedTypes.Where(x => x.IsSubclassOf(_baseType) && !x.IsAbstract); + var foundTypes = assembly.GetTypes().Where(x => x.IsSubclassOf(_baseType) && !x.IsAbstract); foreach (var type in foundTypes) { @@ -51,7 +41,13 @@ public void EnsureCacheIsBuilt() { try { - _cachedTypes.Add(typeName, new CachedTypeInfo(typeName.NotNull(), type, GetPropertyInfo(type))); + _cachedTypes.Add( + typeName, + new CachedTypeInfo( + typeName.NotNull(), + type, + GetPropertyInfo(type), + GetCallbacks(type))); } catch (TypeLoadException) { @@ -66,7 +62,11 @@ public void EnsureCacheIsBuilt() } } - FallbackType = new CachedTypeInfo(_baseType.FullName.NotNull(), _baseType, GetPropertyInfo(_baseType)); + FallbackType = new CachedTypeInfo( + _baseType.FullName.NotNull(), + _baseType, + GetPropertyInfo(_baseType), + GetCallbacks(_baseType)); } public CachedTypeInfo GetType(string speckleType) @@ -76,7 +76,7 @@ public CachedTypeInfo GetType(string speckleType) int start; // _defaultCachedType should be created by now otherwise we should explode - CachedTypeInfo cachedType = FallbackType.NotNull(); + CachedTypeInfo? cachedType; do { @@ -96,19 +96,19 @@ public CachedTypeInfo GetType(string speckleType) if (_cachedTypes.TryGetValue(typeName, out cachedType)) { - return cachedType; + return cachedType.NotNull(); } var lastPeriod = typeName.LastIndexOf('.'); typeName = typeName.Insert(1 + lastPeriod, "Deprecated."); if (_cachedTypes.TryGetValue(typeName, out cachedType)) { - return cachedType; + return cachedType.NotNull(); } } while (start >= 0); // why the hell is this moaning about it being null? - return cachedType; + return FallbackType.NotNull(); } private Dictionary GetPropertyInfo(Type type) @@ -123,4 +123,23 @@ public CachedTypeInfo GetType(string speckleType) return propertyMap; } + + private List GetCallbacks(Type type) + { + List callbacks = new(); + + MethodInfo[] methods = type.GetMethods(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); + foreach (MethodInfo method in methods) + { + List onDeserializedAttributes = method + .GetCustomAttributes(true) + .ToList(); + if (onDeserializedAttributes.Count > 0) + { + callbacks.Add(method); + } + } + + return callbacks; + } } diff --git a/src/Speckle.Core/Serialisation/TypeCache/CachedTypeInfo.cs b/src/Speckle.Core/Serialisation/TypeCache/CachedTypeInfo.cs new file mode 100644 index 0000000..31e4ffe --- /dev/null +++ b/src/Speckle.Core/Serialisation/TypeCache/CachedTypeInfo.cs @@ -0,0 +1,24 @@ +using System.Collections.Immutable; +using System.Reflection; + +namespace Speckle.Core.Serialisation.TypeCache; + +public sealed class CachedTypeInfo +{ + public string Key { get; private set; } + public Type Type { get; private set; } + public ImmutableDictionary Props { get; private set; } + public ImmutableList Callbacks { get; private set; } + + public CachedTypeInfo( + string key, + Type type, + Dictionary props, + List callbacks) + { + Key = key; + Type = type; + Props = props.ToImmutableDictionary(); + Callbacks = callbacks.ToImmutableList(); + } +} diff --git a/src/Speckle.Core/Serialisation/TypeCache/ITypeCache.cs b/src/Speckle.Core/Serialisation/TypeCache/ITypeCache.cs new file mode 100644 index 0000000..3433088 --- /dev/null +++ b/src/Speckle.Core/Serialisation/TypeCache/ITypeCache.cs @@ -0,0 +1,7 @@ +namespace Speckle.Core.Serialisation.TypeCache; + +public interface ITypeCache +{ + void EnsureCacheIsBuilt(); + CachedTypeInfo GetType(string speckelType); +} diff --git a/src/Speckle.Objects/ObjectsKit.cs b/src/Speckle.Objects/ObjectsKit.cs index 8962da1..020062e 100644 --- a/src/Speckle.Objects/ObjectsKit.cs +++ b/src/Speckle.Objects/ObjectsKit.cs @@ -41,8 +41,13 @@ public static string ObjectsFolder public string WebsiteOrEmail => "https://speckle.systems"; /// - public IEnumerable Types => - Assembly.GetExecutingAssembly().GetTypes().Where(t => t.IsSubclassOf(typeof(Base)) && !t.IsAbstract); + public IEnumerable Types + { + get + { + return Assembly.GetExecutingAssembly().GetTypes().Where(t => t.IsSubclassOf(typeof(Base)) && !t.IsAbstract); + } + } /// public IEnumerable Converters => _converters ??= GetAvailableConverters(); diff --git a/src/Speckle.Objects/ObjectsTypeCache.cs b/src/Speckle.Objects/ObjectsTypeCache.cs new file mode 100644 index 0000000..b5ee0df --- /dev/null +++ b/src/Speckle.Objects/ObjectsTypeCache.cs @@ -0,0 +1,9 @@ +using System.Diagnostics; +using System.Reflection; +using Speckle.Core.Models; +using Speckle.Core.Serialisation.TypeCache; + +namespace Speckle.Objects; + +public sealed class ObjectsTypeCache() : AbstractTypeCache(new[] { typeof(ObjectsTypeCache).Assembly }, typeof(Base)); + From 2a0bac962e4288e842e6d919c471c73701cf993e Mon Sep 17 00:00:00 2001 From: Ian Hawley Date: Sun, 30 Jun 2024 23:49:28 +0100 Subject: [PATCH 06/18] - Removed obsolete traversal methods - Removed most Kit references --- .../Kits/ConverterInterfaces/IFinalizable.cs | 6 - src/Speckle.Core/Kits/Exceptions.cs | 29 -- src/Speckle.Core/Kits/ISpeckleConverter.cs | 152 -------- src/Speckle.Core/Kits/ISpeckleKit.cs | 48 --- src/Speckle.Core/Kits/KitDeclaration.cs | 30 -- src/Speckle.Core/Kits/KitManager.cs | 339 ------------------ .../Models/GraphTraversal/DefaultTraversal.cs | 98 ----- src/Speckle.Objects/ObjectsKit.cs | 153 -------- 8 files changed, 855 deletions(-) delete mode 100644 src/Speckle.Core/Kits/ConverterInterfaces/IFinalizable.cs delete mode 100644 src/Speckle.Core/Kits/ISpeckleConverter.cs delete mode 100644 src/Speckle.Core/Kits/ISpeckleKit.cs delete mode 100644 src/Speckle.Core/Kits/KitDeclaration.cs delete mode 100644 src/Speckle.Core/Kits/KitManager.cs delete mode 100644 src/Speckle.Objects/ObjectsKit.cs diff --git a/src/Speckle.Core/Kits/ConverterInterfaces/IFinalizable.cs b/src/Speckle.Core/Kits/ConverterInterfaces/IFinalizable.cs deleted file mode 100644 index 7f10c2f..0000000 --- a/src/Speckle.Core/Kits/ConverterInterfaces/IFinalizable.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace Speckle.Core.Kits.ConverterInterfaces; - -public interface IFinalizable -{ - void FinalizeConversion(); -} diff --git a/src/Speckle.Core/Kits/Exceptions.cs b/src/Speckle.Core/Kits/Exceptions.cs index 6e3c4d4..b269ce3 100644 --- a/src/Speckle.Core/Kits/Exceptions.cs +++ b/src/Speckle.Core/Kits/Exceptions.cs @@ -3,35 +3,6 @@ namespace Speckle.Core.Kits; -/// -/// Exception thrown when an fails to load/initialise -/// -/// -/// Does NOT inherit from , because this usage of this exception is not dependent on Speckle Data (user data) -/// Ideally, this exception should contain a meaningful message, and a reference to the -/// -public class KitException : Exception -{ - /// - /// A reference to the that failed to perform - /// - public ISpeckleKit? Kit { get; } - - public KitException(string? message, ISpeckleKit? kit, Exception? innerException = null) - : base(message, innerException) - { - Kit = kit; - } - - public KitException() { } - - public KitException(string? message) - : base(message) { } - - public KitException(string? message, Exception? innerException) - : base(message, innerException) { } -} - /// /// Exception thrown when conversion of an object was not successful /// diff --git a/src/Speckle.Core/Kits/ISpeckleConverter.cs b/src/Speckle.Core/Kits/ISpeckleConverter.cs deleted file mode 100644 index c7ba36c..0000000 --- a/src/Speckle.Core/Kits/ISpeckleConverter.cs +++ /dev/null @@ -1,152 +0,0 @@ -#nullable disable -using System.Collections.Generic; -using Speckle.Core.Models; - -namespace Speckle.Core.Kits; - -public interface ISpeckleConverter -{ - string Description { get; } - string Name { get; } - string Author { get; } - string WebsiteOrEmail { get; } - - /// - /// Keeps track of the conversion process - /// - public ProgressReport Report { get; } - - /// - /// Decides what to do when an element being received already exists - /// - public ReceiveMode ReceiveMode { get; set; } - - /// - /// Converts a native object to a Speckle one - /// - /// Native object to convert - /// - /// - public Base ConvertToSpeckle(object value); - - /// - /// Converts a list of objects to Speckle. - /// - /// - /// - /// - public List ConvertToSpeckle(List values); - - /// - /// Checks if it can convert a native object to a Speckle one - /// - /// Native object to convert - /// - public bool CanConvertToSpeckle(object value); - - /// - /// Converts a Speckle object to a native one - /// - /// Speckle object to convert - /// - /// - public object ConvertToNative(Base value); - - /// - /// Converts a list of Speckle objects to a native ones. - /// - /// - /// - /// - public List ConvertToNative(List values); - - /// - /// Converts a given speckle objects as a generic native object. - /// This should assume has been called and returned True, - /// or call it within this method's implementation to ensure non-displayable objects are gracefully handled. - /// - /// - /// This method should not try to convert an object to it's native representation (i.e Speckle Wall -> Wall), - /// but rather use the 'displayValue' of that wall to create a geometrically correct representation of that object - /// in the native application. - /// An object may be able to be converted both with and . - /// In this case, deciding which to use is dependent on each connector developer. - /// Preferably, should be used as a fallback to the logic. - /// - /// Speckle object to convert - /// The native object that resulted after converting the input - public object ConvertToNativeDisplayable(Base value); - - /// - /// Checks if it can convert a Speckle object to a native one - /// - /// Speckle object to convert - /// - public bool CanConvertToNative(Base value); - - /// - /// Checks to verify if a given object is: 1) displayable and 2) can be supported for conversion to the native application. - /// An object is considered "displayable" if it has a 'displayValue' property (defined in its class or dynamically attached to it, detached or not). - /// - /// - /// An object may return "True" for both and - /// In this case, deciding which to use is dependent on each connector developer. - /// Preferably, should be used as a fallback to the logic. - /// Objects found in the 'displayValue' property are assumed to be universally convertible by all converters and the viewer, but are not guaranteed to be so. - /// - /// Speckle object to convert - /// True if the object is "displayable" and the converter supports native conversion of the given speckle object in particular. - public bool CanConvertToNativeDisplayable(Base value); - - /// - /// Returns a list of applications serviced by this converter - /// - /// - public IEnumerable GetServicedApplications(); - - /// - /// Sets the application document that the converter is targeting - /// - /// The current application document - public void SetContextDocument(object doc); - - /// - /// Some converters need to know which other objects are being converted, in order to sort relationships between them (ie, Revit). Use this method to set them. - /// - /// - public void SetContextObjects(List objects); - - /// - /// Some converters need to know which objects have been converted before in order to update them (ie, Revit). Use this method to set them. - /// - /// - public void SetPreviousContextObjects(List objects); - - /// - /// Some converters need to be able to receive some settings to modify their internal behaviour (i.e. Rhino's Brep Meshing options). Use this method to set them. - /// - /// The object representing the settings for your converter. - public void SetConverterSettings(object settings); -} - -// NOTE: Do not change the order of the existing ones -/// -/// Receive modes indicate what to do and not do when receiving objects -/// -public enum ReceiveMode -{ - /// - /// Attemts updating previously received objects by ID, deletes previously received objects that do not exist anymore and creates new ones - /// - Update, - - /// - /// Always creates new objects - /// - Create, - - /// - /// Ignores updating previously received objects and does not attempt updating or deleting them, creates new objects - /// - Ignore -} diff --git a/src/Speckle.Core/Kits/ISpeckleKit.cs b/src/Speckle.Core/Kits/ISpeckleKit.cs deleted file mode 100644 index c784ab0..0000000 --- a/src/Speckle.Core/Kits/ISpeckleKit.cs +++ /dev/null @@ -1,48 +0,0 @@ -using System; -using System.Collections.Generic; - -namespace Speckle.Core.Kits; - -/// -/// Defines the basic interface for creating a "Speckle Kit" -/// -public interface ISpeckleKit -{ - /// - /// Gets all the object types (the object model) provided by this kit. - /// - IEnumerable Types { get; } - - /// - /// Gets all available converters for this Kit. - /// - IEnumerable Converters { get; } - - /// - /// Gets this Kit's description. - /// - string Description { get; } - - /// - /// Gets this Kit's name. - /// - string Name { get; } - - /// - /// Gets this Kit's author. - /// - string Author { get; } - - /// - /// Gets the website (or email) to contact the Kit's author. - /// - string WebsiteOrEmail { get; } - - /// - /// Tries to load a converter for a specific . - /// - /// The host app string for which a is desired. see - /// The converter for the specific - /// Thrown if the requested converter failed to load - public ISpeckleConverter LoadConverter(string app); -} diff --git a/src/Speckle.Core/Kits/KitDeclaration.cs b/src/Speckle.Core/Kits/KitDeclaration.cs deleted file mode 100644 index abf8bc3..0000000 --- a/src/Speckle.Core/Kits/KitDeclaration.cs +++ /dev/null @@ -1,30 +0,0 @@ -#nullable disable -using System; -using System.Collections.Generic; -using System.Linq; -using Speckle.Core.Models; - -namespace Speckle.Core.Kits; - -/// -/// Needed so we can properly deserialize all the Base-derived objects from Speckle.Core itself. -/// -public sealed class CoreKit : ISpeckleKit -{ - public IEnumerable Types => GetType().Assembly.GetTypes().Where(type => type.IsSubclassOf(typeof(Base))); - - public string Description => "Base Speckle models for revisions, streams, etc."; - - public string Name => nameof(CoreKit); - - public string Author => "Dimitrie"; - - public string WebsiteOrEmail => "hello@speckle.systems"; - - public IEnumerable Converters => new List(); - - public ISpeckleConverter LoadConverter(string app) - { - return null; - } -} diff --git a/src/Speckle.Core/Kits/KitManager.cs b/src/Speckle.Core/Kits/KitManager.cs deleted file mode 100644 index 0517445..0000000 --- a/src/Speckle.Core/Kits/KitManager.cs +++ /dev/null @@ -1,339 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Reflection; -using Speckle.Core.Helpers; -using Speckle.Core.Logging; -using Speckle.Core.Models; - -namespace Speckle.Core.Kits; - -public static class KitManager -{ - private static string? s_kitsFolder; - - public static readonly AssemblyName SpeckleAssemblyName = typeof(Base).GetTypeInfo().Assembly.GetName(); - - private static readonly Dictionary s_speckleKits = new(); - - private static List s_availableTypes = new(); - - private static bool s_initialized; - - /// - /// Local installations store kits in C:\Users\USERNAME\AppData\Roaming\Speckle\Kits - /// Admin/System-wide installations in C:\ProgramData\Speckle\Kits - /// - public static string KitsFolder - { - get => s_kitsFolder ??= SpecklePathProvider.KitsFolderPath; - set => s_kitsFolder = value; - } - - /// - /// Returns a list of all the kits found on this user's device. - /// - public static IEnumerable Kits - { - get - { - Initialize(); - return s_speckleKits.Values.Where(v => v != null); //NOTE: null check here should be unnecessary - } - } - - /// - /// Returns a list of all the types found in all the kits on this user's device. - /// - public static IEnumerable Types - { - get - { - Initialize(); - return s_availableTypes; - } - } - - /// - /// Checks whether a specific kit exists. - /// - /// - /// - public static bool HasKit(string assemblyFullName) - { - Initialize(); - return s_speckleKits.ContainsKey(assemblyFullName); - } - - /// - /// Gets a specific kit. - /// - /// - /// - public static ISpeckleKit GetKit(string assemblyFullName) - { - Initialize(); - return s_speckleKits[assemblyFullName]; - } - - /// - /// Gets the default Speckle provided kit, "Objects". - /// - /// - public static ISpeckleKit GetDefaultKit() - { - Initialize(); - return s_speckleKits.First(kvp => kvp.Value.Name == "Objects").Value; - } - - /// - /// Returns all the kits with potential converters for the software app. - /// - /// - /// - public static IEnumerable GetKitsWithConvertersForApp(string app) - { - foreach (var kit in Kits) - { - if (kit.Converters.Contains(app)) - { - yield return kit; - } - } - } - - /// - /// Tells the kit manager to initialise from a specific location. - /// - /// - public static void Initialize(string kitFolderLocation) - { - if (s_initialized) - { - SpeckleLog.Logger.Error("{objectType} is already initialised", typeof(KitManager)); - throw new SpeckleException( - "The kit manager has already been initialised. Make sure you call this method earlier in your code!" - ); - } - - KitsFolder = kitFolderLocation; - Load(); - s_initialized = true; - } - - #region Private Methods - - private static void Initialize() - { - if (!s_initialized) - { - Load(); - s_initialized = true; - } - } - - private static void Load() - { - SpeckleLog.Logger.Information("Initializing Kit Manager in {KitsFolder}", SpecklePathProvider.KitsFolderPath); - - GetLoadedSpeckleReferencingAssemblies(); - LoadSpeckleReferencingAssemblies(); - - s_availableTypes = s_speckleKits - .Where(kit => kit.Value != null) //Null check should be unnecessary - .SelectMany(kit => kit.Value.Types) - .ToList(); - } - - // recursive search for referenced assemblies - public static List GetReferencedAssemblies() - { - var returnAssemblies = new List(); - var loadedAssemblies = new HashSet(); - var assembliesToCheck = new Queue(); - - assembliesToCheck.Enqueue(Assembly.GetEntryAssembly()); - - while (assembliesToCheck.Count > 0) - { - var assemblyToCheck = assembliesToCheck.Dequeue(); - - if (assemblyToCheck == null) - { - continue; - } - - foreach (var reference in assemblyToCheck.GetReferencedAssemblies()) - { - // filtering out system dlls - if (reference.FullName.StartsWith("System.")) - { - continue; - } - - if (reference.FullName.StartsWith("Microsoft.")) - { - continue; - } - - if (loadedAssemblies.Contains(reference.FullName)) - { - continue; - } - - Assembly assembly; - try - { - assembly = Assembly.Load(reference); - } - catch (SystemException ex) when (ex is IOException or BadImageFormatException) - { - continue; - } - - assembliesToCheck.Enqueue(assembly); - loadedAssemblies.Add(reference.FullName); - returnAssemblies.Add(assembly); - } - } - - return returnAssemblies; - } - - private static void GetLoadedSpeckleReferencingAssemblies() - { - List assemblies = AppDomain.CurrentDomain.GetAssemblies().ToList(); - assemblies.AddRange(GetReferencedAssemblies()); - - foreach (var assembly in assemblies) - { - if (assembly.IsDynamic || assembly.ReflectionOnly) - { - continue; - } - - if (!assembly.IsReferencing(SpeckleAssemblyName)) - { - continue; - } - - if (s_speckleKits.ContainsKey(assembly.FullName)) - { - continue; - } - - var kitClass = GetKitClass(assembly); - if (kitClass == null) - { - continue; - } - - if (Activator.CreateInstance(kitClass) is ISpeckleKit speckleKit) - { - s_speckleKits.Add(assembly.FullName, speckleKit); - } - } - } - - private static void LoadSpeckleReferencingAssemblies() - { - if (!Directory.Exists(KitsFolder)) - { - return; - } - - var directories = Directory.GetDirectories(KitsFolder); - - foreach (var directory in directories) - { - foreach (var assemblyPath in Directory.EnumerateFiles(directory, "*.dll")) - { - var unloadedAssemblyName = SafeGetAssemblyName(assemblyPath); - - if (unloadedAssemblyName == null) - { - continue; - } - - try - { - var assembly = Assembly.LoadFrom(assemblyPath); - var kitClass = GetKitClass(assembly); - if (assembly.IsReferencing(SpeckleAssemblyName) && kitClass != null) - { - if (!s_speckleKits.ContainsKey(assembly.FullName)) - { - if (Activator.CreateInstance(kitClass) is ISpeckleKit speckleKit) - { - s_speckleKits.Add(assembly.FullName, speckleKit); - } - } - } - } - catch (FileLoadException) { } - catch (BadImageFormatException) { } - } - } - } - - private static Type? GetKitClass(Assembly assembly) - { - try - { - var kitClass = assembly - .GetTypes() - .FirstOrDefault(type => - { - return type.GetInterfaces().Any(iface => iface.Name == nameof(ISpeckleKit)); - }); - - return kitClass; - } - catch (ReflectionTypeLoadException) - { - return null; - } - } - - private static AssemblyName? SafeGetAssemblyName(string? assemblyPath) - { - try - { - return AssemblyName.GetAssemblyName(assemblyPath); - } - catch (Exception ex) when (ex is ArgumentException or IOException or BadImageFormatException) - { - return null; - } - } - - #endregion -} - -internal static class AssemblyExtensions -{ - /// - /// Indicates if a given assembly references another which is identified by its name. - /// - /// The assembly which will be probed. - /// The reference assembly name. - /// A boolean value indicating if there is a reference. - public static bool IsReferencing(this Assembly assembly, AssemblyName referenceName) - { - if (AssemblyName.ReferenceMatchesDefinition(assembly.GetName(), referenceName)) - { - return true; - } - - foreach (var referencedAssemblyName in assembly.GetReferencedAssemblies()) - { - if (AssemblyName.ReferenceMatchesDefinition(referencedAssemblyName, referenceName)) - { - return true; - } - } - - return false; - } -} diff --git a/src/Speckle.Core/Models/GraphTraversal/DefaultTraversal.cs b/src/Speckle.Core/Models/GraphTraversal/DefaultTraversal.cs index 885b031..4e87e19 100644 --- a/src/Speckle.Core/Models/GraphTraversal/DefaultTraversal.cs +++ b/src/Speckle.Core/Models/GraphTraversal/DefaultTraversal.cs @@ -111,102 +111,4 @@ public static IEnumerable DisplayValueAndElementsAliases(Base _) } #endregion - - #region Legacy function varients - - [Obsolete("Renamed to " + nameof(ElementsPropAliases))] - [SuppressMessage("Style", "IDE1006:Naming Styles", Justification = "Obsolete")] - public static IReadOnlyList elementsPropAliases => ElementsPropAliases; - - [Obsolete("Renamed to " + nameof(DisplayValuePropAliases))] - [SuppressMessage("Style", "IDE1006:Naming Styles", Justification = "Obsolete")] - public static IReadOnlyList displayValuePropAliases => DisplayValuePropAliases; - - [Obsolete("Renamed to " + nameof(DefinitionPropAliases))] - [SuppressMessage("Style", "IDE1006:Naming Styles", Justification = "Obsolete")] - public static IReadOnlyList definitionPropAliases => DefinitionPropAliases; - - [Obsolete("Renamed to " + nameof(GeometryPropAliases))] - [SuppressMessage("Style", "IDE1006:Naming Styles", Justification = "Obsolete")] - public static IReadOnlyList geometryPropAliases => GeometryPropAliases; - - [Obsolete("Renamed to " + nameof(DisplayValueAndElementsPropAliases))] - [SuppressMessage("Style", "IDE1006:Naming Styles", Justification = "Obsolete")] - [SuppressMessage("Performance", "CA1819:Properties should not return arrays", Justification = "Obsolete")] - public static string[] displayValueAndElementsPropAliases => DisplayValueAndElementsPropAliases; - - /// - /// - /// - /// - [Obsolete($"Consider using {nameof(CreateTraversalFunc)}")] - public static GraphTraversal CreateTraverseFunc(ISpeckleConverter converter) - { - return CreateLegacyTraverseFunc(converter.CanConvertToNative); - } - - /// - /// Legacy traversal rule that was dependent on the converter - /// - /// - /// Treats convertable objects and objects with displayValues as "convertable" such that only elements and dynamic props will be traversed - /// New code should use instead. - /// - /// - /// - [Obsolete($"Consider using {nameof(CreateTraversalFunc)}")] - public static GraphTraversal CreateLegacyTraverseFunc(Func canConvertToNative) - { - var convertableRule = TraversalRule - .NewTraversalRule() - .When(b => canConvertToNative(b)) - .When(HasDisplayValue) - .ContinueTraversing(_ => ElementsPropAliases); - - return new GraphTraversal(convertableRule, s_ignoreResultsRule, DefaultRule); - } - - /// - /// Traverses until finds a convertable object then HALTS deeper traversal - /// - /// - /// The DUI2 Revit connector does traversal, - /// so this traversal is a shallow traversal for directly convertable objects, - /// and a deep traversal for all other types - /// New code should use instead. - /// - /// - /// - [Obsolete($"Consider using {nameof(CreateTraversalFunc)}")] - public static GraphTraversal CreateRevitTraversalFunc(ISpeckleConverter converter) - { - var convertableRule = TraversalRule - .NewTraversalRule() - .When(converter.CanConvertToNative) - .When(HasDisplayValue) - .ContinueTraversing(None); - - return new GraphTraversal(convertableRule, s_ignoreResultsRule, DefaultRule); - } - - /// - /// Traverses until finds a convertable object (or fallback) then traverses members - /// - /// - /// New code should use instead. - /// - /// - /// - [Obsolete($"Consider using {nameof(CreateTraversalFunc)}")] - public static GraphTraversal CreateBIMTraverseFunc(ISpeckleConverter converter) - { - var bimElementRule = TraversalRule - .NewTraversalRule() - .When(converter.CanConvertToNative) - .ContinueTraversing(ElementsAliases); - - return new GraphTraversal(bimElementRule, s_ignoreResultsRule, DefaultRule); - } - - #endregion } diff --git a/src/Speckle.Objects/ObjectsKit.cs b/src/Speckle.Objects/ObjectsKit.cs deleted file mode 100644 index 020062e..0000000 --- a/src/Speckle.Objects/ObjectsKit.cs +++ /dev/null @@ -1,153 +0,0 @@ -using System.Reflection; -using Speckle.Core.Helpers; -using Speckle.Core.Kits; -using Speckle.Core.Logging; -using Speckle.Core.Models; - -namespace Objects; - -/// -/// The default Speckle Kit -/// -public class ObjectsKit : ISpeckleKit -{ - private static string? s_objectsFolder; - - private readonly Dictionary _loadedConverters = new(); - - private List? _converters; - - /// - /// Local installations store objects in C:\Users\USERNAME\AppData\Roaming\Speckle\Kits\Objects - /// Admin/System-wide installations in C:\ProgramData\Speckle\Kits\Objects - /// - public static string ObjectsFolder - { - get => s_objectsFolder ??= SpecklePathProvider.ObjectsFolderPath; - [Obsolete("Use " + nameof(SpecklePathProvider.OverrideObjectsFolderName), true)] - set => s_objectsFolder = value; - } - - /// - public string Description => "The default Speckle Kit."; - - /// - public string Name => "Objects"; - - /// - public string Author => "Speckle"; - - /// - public string WebsiteOrEmail => "https://speckle.systems"; - - /// - public IEnumerable Types - { - get - { - return Assembly.GetExecutingAssembly().GetTypes().Where(t => t.IsSubclassOf(typeof(Base)) && !t.IsAbstract); - } - } - - /// - public IEnumerable Converters => _converters ??= GetAvailableConverters(); - - /// - public ISpeckleConverter LoadConverter(string app) - { - try - { - _converters = GetAvailableConverters(); - if (_loadedConverters.TryGetValue(app, out Type t)) - { - return (ISpeckleConverter)Activator.CreateInstance(t); - } - - var converterInstance = LoadConverterFromDisk(app); - _loadedConverters[app] = converterInstance.GetType(); - - return converterInstance; - } - catch (Exception ex) - { - SpeckleLog.Logger.Fatal(ex, "Failed to load converter for app {app}", app); - throw new KitException($"Failed to load converter for app {app}:\n\n{ex.Message}", this, ex); - } - } - - private static ISpeckleConverter LoadConverterFromDisk(string app) - { - var basePath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); - - var path = Path.Combine(basePath!, $"Objects.Converter.{app}.dll"); - - //fallback to the default folder, in case the Objects.dll was loaded in the app domain for other reasons - if (!File.Exists(path)) - { - path = Path.Combine(ObjectsFolder, $"Objects.Converter.{app}.dll"); - } - - if (!File.Exists(path)) - { - throw new FileNotFoundException($"Converter for {app} was not found in kit {basePath}", path); - } - - AssemblyName assemblyToLoad = AssemblyName.GetAssemblyName(path); - var objects = Assembly.GetExecutingAssembly().GetName(); - - //only get assemblies matching the Major and Minor version of Objects - if (assemblyToLoad.Version.Major != objects.Version.Major || assemblyToLoad.Version.Minor != objects.Version.Minor) - { - throw new SpeckleException( - $"Mismatch between Objects library v{objects.Version} Converter v{assemblyToLoad.Version}.\nEnsure the same 2.x version of Speckle connectors is installed." - ); - } - - var assembly = Assembly.LoadFrom(path); - - var converterInstance = assembly - .GetTypes() - .Where(type => typeof(ISpeckleConverter).IsAssignableFrom(type)) - .Select(type => (ISpeckleConverter)Activator.CreateInstance(type)) - .FirstOrDefault(converter => converter.GetServicedApplications().Contains(app)); - - if (converterInstance == null) - { - throw new SpeckleException($"No suitable converter instance found for {app}"); - } - - SpeckleLog - .Logger.ForContext() - .ForContext("basePath", basePath) - .ForContext("app", app) - .Information("Converter {converterName} successfully loaded from {path}", converterInstance.Name, path); - - return converterInstance; - } - - public List GetAvailableConverters() - { - var basePath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); - var allConverters = Directory.EnumerateFiles(basePath!, "Objects.Converter.*.dll").ToArray(); - - //fallback to the default folder, in case the Objects.dll was loaded in the app domain for other reasons - if (allConverters.Length == 0) - { - allConverters = Directory.EnumerateFiles(ObjectsFolder, "Objects.Converter.*.dll").ToArray(); - } - - //only get assemblies matching the Major and Minor version of Objects - var objects = Assembly.GetExecutingAssembly().GetName(); - var availableConverters = new List(); - foreach (var converter in allConverters) - { - AssemblyName assemblyName = AssemblyName.GetAssemblyName(converter); - if (assemblyName.Version.Major == objects.Version.Major && assemblyName.Version.Minor == objects.Version.Minor) - { - availableConverters.Add(converter); - } - } - - return availableConverters.Select(dllPath => dllPath.Split('.').Reverse().ElementAt(1)).ToList(); - } -} From 6a0020f9214874d83913565ee28571a006273b08 Mon Sep 17 00:00:00 2001 From: Ian Hawley Date: Tue, 2 Jul 2024 19:31:37 +0100 Subject: [PATCH 07/18] the type we deserialise to are now deserialising to a matching version if it exists next steps: - get the version from the commit - pick a version to use if there isn't one... what would this be? hmmm should be oldest version? want to avoid duplicating the world at this stage :D - add in the conversion logic, registering the type upgrader and picking upgrades to return the actual type we want to load --- .../Serialisation/BaseObjectDeserializerV2.cs | 33 ++-- .../Serialisation/BaseObjectSerializerV2.cs | 4 +- .../Serialisation/SerializationConstants.cs | 10 ++ .../TypeCache/AbstractTypeCache.cs | 151 +++++++++++++++--- .../Serialisation/TypeCache/ITypeCache.cs | 5 +- src/Speckle.Core/Speckle.Core.csproj | 4 - src/Speckle.Core/Transports/ServerV2.cs | 3 +- src/Speckle.Objects/ObjectsTypeCache.cs | 13 +- src/Speckle.Objects/SpeckleSchemaInfo.cs | 4 +- src/Speckle.Objects/Versions/V_0_1_0/Wall.cs | 44 +++++ 10 files changed, 215 insertions(+), 56 deletions(-) create mode 100644 src/Speckle.Core/Serialisation/SerializationConstants.cs create mode 100644 src/Speckle.Objects/Versions/V_0_1_0/Wall.cs diff --git a/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs b/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs index 90703aa..285171a 100644 --- a/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs +++ b/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs @@ -25,11 +25,6 @@ public sealed class BaseObjectDeserializerV2 : ISpeckleDeserializer // id -> Base if already deserialized or id -> Task if was handled by a bg thread private Dictionary? _deserializedObjects; - /// - /// Property that describes the type of the object. - /// - private const string TYPE_DISCRIMINATOR = nameof(Base.speckle_type); - private DeserializationWorkerThreads? _workerThreads; public CancellationToken CancellationToken { get; set; } @@ -136,12 +131,12 @@ private List<(string, int)> GetClosures(string rootObjectJson) List<(string, int)> closureList = new(); JObject doc1 = JObject.Parse(rootObjectJson); - if (!doc1.ContainsKey("__closure")) + if (!doc1.ContainsKey(SerializationConstants.CLOSURE_PROPERTY_NAME)) { return new List<(string, int)>(); } - foreach (JToken prop in doc1["__closure"].NotNull()) + foreach (JToken prop in doc1[SerializationConstants.CLOSURE_PROPERTY_NAME].NotNull()) { string childId = ((JProperty)prop).Name; int childMinDepth = (int)((JProperty)prop).Value; @@ -280,7 +275,7 @@ private List<(string, int)> GetClosures(string rootObjectJson) foreach (JToken propJToken in jObject) { JProperty prop = (JProperty)propJToken; - if (prop.Name == "__closure") + if (prop.Name == SerializationConstants.CLOSURE_PROPERTY_NAME) { continue; } @@ -288,7 +283,7 @@ private List<(string, int)> GetClosures(string rootObjectJson) dict[prop.Name] = ConvertJsonElement(prop.Value); } - if (!dict.TryGetValue(TYPE_DISCRIMINATOR, out object? speckleType)) + if (!dict.TryGetValue(SerializationConstants.TYPE_DISCRIMINATOR, out object? speckleType)) { return dict; } @@ -351,19 +346,19 @@ private List<(string, int)> GetClosures(string rootObjectJson) private Base Dict2Base(Dictionary dictObj) { - string typeName = (string)dictObj[TYPE_DISCRIMINATOR]!; + string typeName = (string)dictObj[SerializationConstants.TYPE_DISCRIMINATOR]!; - // TODO: get the version - - // we find the type - CachedTypeInfo cachedTypeInfo = _typeCache.GetType(typeName); - - // let's figure out whether to deserialise into this type or a legacy type + // TODO: pass the version IN + Version schemaVersion = _typeCache.LoadedSchemaVersion; + + // here we're getting the actual type to deserialise into, this won't be the type we return + CachedTypeInfo cachedTypeInfo = _typeCache.GetMatchedTypeOrLater(typeName, schemaVersion); Base baseObj = (Base) Activator.CreateInstance(cachedTypeInfo.Type); - dictObj.Remove(TYPE_DISCRIMINATOR); - dictObj.Remove("__closure"); + dictObj.Remove(SerializationConstants.TYPE_DISCRIMINATOR); + dictObj.Remove(SerializationConstants.CLOSURE_PROPERTY_NAME); + dictObj.Remove(SerializationConstants.SCHEMA_VERSION); var props = cachedTypeInfo.Props; @@ -411,7 +406,7 @@ private Base Dict2Base(Dictionary dictObj) bb.filePath = bb.GetLocalDestinationPath(BlobStorageFolder); } - // TODO: perform any versioning we need to + // TODO: perform any versioning we need to, to upgrade the type foreach (MethodInfo onDeserialized in onDeserializedCallbacks) { diff --git a/src/Speckle.Core/Serialisation/BaseObjectSerializerV2.cs b/src/Speckle.Core/Serialisation/BaseObjectSerializerV2.cs index 27a1b30..4838c6c 100644 --- a/src/Speckle.Core/Serialisation/BaseObjectSerializerV2.cs +++ b/src/Speckle.Core/Serialisation/BaseObjectSerializerV2.cs @@ -138,7 +138,7 @@ public string Serialize(Base baseObj) { ["speckle_type"] = r.speckle_type, ["referencedId"] = r.referencedId, - ["__closure"] = r.closure + [SerializationConstants.CLOSURE_PROPERTY_NAME] = r.closure }; if (r.closure is not null) { @@ -313,7 +313,7 @@ public string Serialize(Base baseObj) if (closure.Count > 0) { - convertedBase["__closure"] = closure; + convertedBase[SerializationConstants.CLOSURE_PROPERTY_NAME] = closure; } if (computeClosures || inheritedDetachInfo.IsDetachable || baseObj is Blob) diff --git a/src/Speckle.Core/Serialisation/SerializationConstants.cs b/src/Speckle.Core/Serialisation/SerializationConstants.cs new file mode 100644 index 0000000..07e6b2c --- /dev/null +++ b/src/Speckle.Core/Serialisation/SerializationConstants.cs @@ -0,0 +1,10 @@ +using Speckle.Core.Models; + +namespace Speckle.Core.Serialisation; + +public class SerializationConstants +{ + public const string TYPE_DISCRIMINATOR = nameof(Base.speckle_type); + public const string CLOSURE_PROPERTY_NAME = "__closure"; + public const string SCHEMA_VERSION = "__schema_version"; +} diff --git a/src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs b/src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs index 979565d..9039a4f 100644 --- a/src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs +++ b/src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs @@ -1,4 +1,5 @@ using System.Collections.Immutable; +using System.Collections.Specialized; using System.Reflection; using System.Runtime.Serialization; using Speckle.Core.Common; @@ -6,23 +7,43 @@ namespace Speckle.Core.Serialisation.TypeCache; +internal class VersionCache(string type) +{ + public string Type { get; private set; } = type; + public List<(Version, CachedTypeInfo)> Versions => new (); + public CachedTypeInfo? LatestVersion; +} + public abstract class AbstractTypeCache : ITypeCache { private readonly IList _assemblies; private readonly Type _baseType; - private readonly Dictionary _cachedTypes = new(); + private readonly Dictionary _cachedTypes = new(); + public Version LoadedSchemaVersion { get; private set; } + + private readonly string _namespacePrefix; + private readonly string _versionNamespacePrefix; + public CachedTypeInfo? FallbackType { get; private set; } - protected AbstractTypeCache(IEnumerable assemblies, Type baseType) + protected AbstractTypeCache( + IEnumerable assemblies, + Type baseType, + Version loadedSchemaVersion, + string namespacePrefix) { _assemblies = assemblies.ToList(); _baseType = baseType; + LoadedSchemaVersion = loadedSchemaVersion; + _namespacePrefix = namespacePrefix; + _versionNamespacePrefix = $"{_namespacePrefix}.Versions."; // POC: manually including core, not sure of the wisdom of this... _assemblies.Add(typeof(AbstractTypeCache).Assembly); } + // could build in constructor.. but throwing from constructors generally frowned upon public void EnsureCacheIsBuilt() { // POC: need a way to pick our own objects and not the DUI2 objects plus this is a touch weak :pain @@ -34,25 +55,21 @@ public void EnsureCacheIsBuilt() foreach (var type in foundTypes) { - var typeName = type.FullName; + (string typeName, Version? version) = GetTypeNameAndVersion(type.FullName.NotNull()); - // POC: I am not convinced we should have duplicates | plus the original Objects.Dll could be loaded at the same time - if (!_cachedTypes.ContainsKey(typeName)) + try + { + var typeCacheInfo = new CachedTypeInfo( + typeName.NotNull(), + type, + GetPropertyInfo(type), + GetCallbacks(type)); + + CacheType(typeName, version, typeCacheInfo); + } + catch (TypeLoadException) { - try - { - _cachedTypes.Add( - typeName, - new CachedTypeInfo( - typeName.NotNull(), - type, - GetPropertyInfo(type), - GetCallbacks(type))); - } - catch (TypeLoadException) - { - // POC: guard against loading things that cause explosions - } + // POC: guard against loading things that cause explosions } } } @@ -62,6 +79,14 @@ public void EnsureCacheIsBuilt() } } + // TODO: if we load versions but no latest, then this is an error + // future incarnations may permit mutating the type + // i.e. decorate a new type with the name of the legacy name + + + // TODO: order versioned cache by their version to save sorting every version look up + + FallbackType = new CachedTypeInfo( _baseType.FullName.NotNull(), _baseType, @@ -69,14 +94,14 @@ public void EnsureCacheIsBuilt() GetCallbacks(_baseType)); } - public CachedTypeInfo GetType(string speckleType) + public CachedTypeInfo GetMatchedTypeOrLater(string speckleType, Version versionToMatch) { int length = speckleType.Length; int end = length - 1; int start; // _defaultCachedType should be created by now otherwise we should explode - CachedTypeInfo? cachedType; + VersionCache? cachedVersions; do { @@ -94,16 +119,16 @@ public CachedTypeInfo GetType(string speckleType) end = start - 1; } - if (_cachedTypes.TryGetValue(typeName, out cachedType)) + if (_cachedTypes.TryGetValue(typeName, out cachedVersions)) { - return cachedType.NotNull(); + return MatchOrLater(versionToMatch, cachedVersions); } var lastPeriod = typeName.LastIndexOf('.'); typeName = typeName.Insert(1 + lastPeriod, "Deprecated."); - if (_cachedTypes.TryGetValue(typeName, out cachedType)) + if (_cachedTypes.TryGetValue(typeName, out cachedVersions)) { - return cachedType.NotNull(); + return MatchOrLater(versionToMatch, cachedVersions); } } while (start >= 0); @@ -111,6 +136,82 @@ public CachedTypeInfo GetType(string speckleType) return FallbackType.NotNull(); } + private CachedTypeInfo MatchOrLater(Version versionToMatch, VersionCache versionCache) + { + if (versionToMatch == LoadedSchemaVersion) + { + return versionCache.LatestVersion.NotNull(); + } + + // we could search or we can walk. We might be able to optomise this + foreach ((Version version, CachedTypeInfo cachedTypeInfo) cachedVersion in versionCache.Versions) + { + // if it's a match or comes later, then use this + if (cachedVersion.version >= versionToMatch) + { + return cachedVersion.cachedTypeInfo; + } + } + + // if we get here just use the latest version + return versionCache.LatestVersion.NotNull(); + } + + private void CacheType(string typeName, Version version, CachedTypeInfo typeCacheInfo) + { + if (!_cachedTypes.TryGetValue(typeName, out VersionCache versionCache)) + { + versionCache = new VersionCache(typeName); + _cachedTypes[typeName] = versionCache; + } + + // is this the latest? + if (version == LoadedSchemaVersion) + { + // POC: we could be checking for prior existence + versionCache.LatestVersion = typeCacheInfo; + } + else + { + // we should NOT already have one of these + if (versionCache.Versions.Any<(Version version, CachedTypeInfo cache)>(item => item.version == version)) + { + throw new ArgumentException($"Version '{version}' already exists for cached type '{typeName}'"); + } + + versionCache.Versions.Add((version, typeCacheInfo)); + } + } + + private (string name, Version version) GetTypeNameAndVersion(string typeName) + { + // is this versioned at all? + if (typeName.StartsWith(_versionNamespacePrefix)) + { + // this is a version, the next portion is the version number + var nextPeriod = typeName.IndexOf('.', _versionNamespacePrefix.Length); + if (nextPeriod == -1 || nextPeriod == typeName.Length - 1) + { + // period should be found but should not be the last character + throw new ArgumentException($"Loaded type {typeName} appears to be a version but is malformed, expected a '.' and subsequent type name"); + } + + // POC: this is a sort of magic string with structure, I don't mind it because it is coming from the actual code but maybe worth considering + // we assume the version starts V_ hence shunting things 2 chars and replacing the _ with periods + int versionLength = nextPeriod - (_versionNamespacePrefix.Length + 2); + string versionString = + typeName.Substring(_versionNamespacePrefix.Length + 2, versionLength) + .Replace('_', '.'); + + string unversionedTypeName = $"{_namespacePrefix}{typeName.Substring(nextPeriod)}"; + + // we have to remove the Versionsless prefix + return (unversionedTypeName, new Version(versionString)); + } + + return (typeName, LoadedSchemaVersion); + } + private Dictionary GetPropertyInfo(Type type) { Dictionary propertyMap = new(); diff --git a/src/Speckle.Core/Serialisation/TypeCache/ITypeCache.cs b/src/Speckle.Core/Serialisation/TypeCache/ITypeCache.cs index 3433088..28a2303 100644 --- a/src/Speckle.Core/Serialisation/TypeCache/ITypeCache.cs +++ b/src/Speckle.Core/Serialisation/TypeCache/ITypeCache.cs @@ -3,5 +3,8 @@ namespace Speckle.Core.Serialisation.TypeCache; public interface ITypeCache { void EnsureCacheIsBuilt(); - CachedTypeInfo GetType(string speckelType); + + CachedTypeInfo GetMatchedTypeOrLater(string speckleType, Version versionToMatch); + + Version LoadedSchemaVersion { get; } } diff --git a/src/Speckle.Core/Speckle.Core.csproj b/src/Speckle.Core/Speckle.Core.csproj index 2c9df1a..0c499dc 100644 --- a/src/Speckle.Core/Speckle.Core.csproj +++ b/src/Speckle.Core/Speckle.Core.csproj @@ -36,9 +36,5 @@ - - - - diff --git a/src/Speckle.Core/Transports/ServerV2.cs b/src/Speckle.Core/Transports/ServerV2.cs index ff1baa2..4bfcd3a 100644 --- a/src/Speckle.Core/Transports/ServerV2.cs +++ b/src/Speckle.Core/Transports/ServerV2.cs @@ -9,6 +9,7 @@ using Speckle.Core.Helpers; using Speckle.Core.Logging; using Speckle.Core.Models; +using Speckle.Core.Serialisation; using Speckle.Core.Transports.ServerUtils; using Speckle.Newtonsoft.Json.Linq; @@ -326,7 +327,7 @@ private static IList ParseChildrenIds(string json) List childrenIds = new(); JObject doc1 = JObject.Parse(json); - JToken? closures = doc1["__closure"]; + JToken? closures = doc1[SerializationConstants.CLOSURE_PROPERTY_NAME]; if (closures == null) { return Array.Empty(); diff --git a/src/Speckle.Objects/ObjectsTypeCache.cs b/src/Speckle.Objects/ObjectsTypeCache.cs index b5ee0df..445a7e5 100644 --- a/src/Speckle.Objects/ObjectsTypeCache.cs +++ b/src/Speckle.Objects/ObjectsTypeCache.cs @@ -1,9 +1,18 @@ using System.Diagnostics; using System.Reflection; +using Objects; using Speckle.Core.Models; using Speckle.Core.Serialisation.TypeCache; namespace Speckle.Objects; -public sealed class ObjectsTypeCache() : AbstractTypeCache(new[] { typeof(ObjectsTypeCache).Assembly }, typeof(Base)); - +public sealed class ObjectsTypeCache : AbstractTypeCache +{ + public ObjectsTypeCache(Version? version = null) + : base(new[] { typeof(ObjectsTypeCache).Assembly }, + typeof(Base), + version ?? SpeckleSchemaInfo.Version, + "Objects") + { + } +} diff --git a/src/Speckle.Objects/SpeckleSchemaInfo.cs b/src/Speckle.Objects/SpeckleSchemaInfo.cs index b09699b..ed930ac 100644 --- a/src/Speckle.Objects/SpeckleSchemaInfo.cs +++ b/src/Speckle.Objects/SpeckleSchemaInfo.cs @@ -1,6 +1,6 @@ namespace Objects; -public class SpeckleSchemaInfo +public static class SpeckleSchemaInfo { - public readonly Version Version = new(3, 0, 0); + public static readonly Version Version = new(0, 2, 0); } diff --git a/src/Speckle.Objects/Versions/V_0_1_0/Wall.cs b/src/Speckle.Objects/Versions/V_0_1_0/Wall.cs new file mode 100644 index 0000000..469157b --- /dev/null +++ b/src/Speckle.Objects/Versions/V_0_1_0/Wall.cs @@ -0,0 +1,44 @@ +using Objects; +using Objects.BuiltElements; +using Objects.Geometry; +using Speckle.Core.Kits; +using Speckle.Core.Models; + +namespace Objects.Versions.V_0_1_0; + +public class Wall : Base, IDisplayValue> +{ + public Wall() { } + + /// + /// SchemaBuilder constructor for a Speckle wall + /// + /// + /// + /// + /// Assign units when using this constructor due to param + [SchemaInfo("Wall", "Creates a Speckle wall", "BIM", "Architecture")] + public Wall( + double height, + [SchemaMainParam] ICurve baseLine, + [SchemaParamInfo("Any nested elements that this wall might have")] List? elements = null + ) + { + this.height = height; + this.baseLine = baseLine; + this.elements = elements; + } + + public double height { get; set; } + + [DetachProperty] + public List? elements { get; set; } + + public ICurve baseLine { get; set; } + public virtual Level? level { get; internal set; } + + public string units { get; set; } + + [DetachProperty] + public List displayValue { get; set; } +} From a21a84cf6a798d558ae0d549c78479a561ccd96a Mon Sep 17 00:00:00 2001 From: Ian Hawley Date: Tue, 2 Jul 2024 23:39:31 +0100 Subject: [PATCH 08/18] - schema version now coming from the commit - being passed along to Dict2Base() where it can be used to pick the right deserialisation target --- src/Speckle.Core/Api/GraphQL/Models.cs | 4 ++ src/Speckle.Core/Api/Helpers.cs | 2 + .../Api/Operations/Operations.Receive.cs | 3 +- .../Api/Operations/Operations.Serialize.cs | 4 +- .../SchemaVersioning/IObjectSchemaUpgrade.cs | 51 +++++++++++++++++++ .../Serialisation/BaseObjectDeserializerV2.cs | 11 ++-- .../TypeCache/AbstractTypeCache.cs | 39 ++++++++++++-- .../Serialisation/TypeCache/CachedTypeInfo.cs | 6 +-- .../V_0_1_0/{ => BuiltElements}/Wall.cs | 2 +- 9 files changed, 105 insertions(+), 17 deletions(-) create mode 100644 src/Speckle.Core/SchemaVersioning/IObjectSchemaUpgrade.cs rename src/Speckle.Objects/Versions/V_0_1_0/{ => BuiltElements}/Wall.cs (95%) diff --git a/src/Speckle.Core/Api/GraphQL/Models.cs b/src/Speckle.Core/Api/GraphQL/Models.cs index 9b60931..c659d82 100644 --- a/src/Speckle.Core/Api/GraphQL/Models.cs +++ b/src/Speckle.Core/Api/GraphQL/Models.cs @@ -207,6 +207,10 @@ public class Commit public int totalChildrenCount { get; set; } public List parents { get; set; } + // should be nullable but is currently disabled here + // null schema == this is old AF + public string SchemaVersion { get; set; } + public override string ToString() { return $"Commit ({message} | {id})"; diff --git a/src/Speckle.Core/Api/Helpers.cs b/src/Speckle.Core/Api/Helpers.cs index e110119..e8c3a0f 100644 --- a/src/Speckle.Core/Api/Helpers.cs +++ b/src/Speckle.Core/Api/Helpers.cs @@ -37,6 +37,7 @@ public static class Helpers public static async Task Receive( string stream, ITypeCache typeCache, + System.Version schemaVersion, Account account = null, Action> onProgressAction = null, Action onTotalChildrenCountKnown = null @@ -110,6 +111,7 @@ public static class Helpers .Receive( objectId, typeCache, + schemaVersion, transport, onProgressAction: onProgressAction, onTotalChildrenCountKnown: onTotalChildrenCountKnown diff --git a/src/Speckle.Core/Api/Operations/Operations.Receive.cs b/src/Speckle.Core/Api/Operations/Operations.Receive.cs index 0c597ee..fe2ca59 100644 --- a/src/Speckle.Core/Api/Operations/Operations.Receive.cs +++ b/src/Speckle.Core/Api/Operations/Operations.Receive.cs @@ -40,6 +40,7 @@ public static partial class Operations public static async Task Receive( string objectId, ITypeCache typeCache, + System.Version schemaVersion, ITransport? remoteTransport = null, ITransport? localTransport = null, Action>? onProgressAction = null, @@ -64,7 +65,7 @@ public static partial class Operations // Setup Serializer BaseObjectDeserializerV2 serializerV2 = - new(typeCache) + new(typeCache, schemaVersion) { ReadTransport = localTransport, OnProgressAction = internalProgressAction, diff --git a/src/Speckle.Core/Api/Operations/Operations.Serialize.cs b/src/Speckle.Core/Api/Operations/Operations.Serialize.cs index 87a206f..8ff415f 100644 --- a/src/Speckle.Core/Api/Operations/Operations.Serialize.cs +++ b/src/Speckle.Core/Api/Operations/Operations.Serialize.cs @@ -37,9 +37,9 @@ public static string Serialize(Base value, CancellationToken cancellationToken = /// was not valid JSON /// cannot be deserialised to type /// contains closure references (see Remarks) - public static Base Deserialize(string value, ITypeCache typeCache, CancellationToken cancellationToken = default) + public static Base Deserialize(string value, ITypeCache typeCache, System.Version payloadSchemaVersion, CancellationToken cancellationToken = default) { - var deserializer = new BaseObjectDeserializerV2(typeCache) { CancellationToken = cancellationToken }; + var deserializer = new BaseObjectDeserializerV2(typeCache, payloadSchemaVersion) { CancellationToken = cancellationToken }; return deserializer.Deserialize(value); } } diff --git a/src/Speckle.Core/SchemaVersioning/IObjectSchemaUpgrade.cs b/src/Speckle.Core/SchemaVersioning/IObjectSchemaUpgrade.cs new file mode 100644 index 0000000..7fdd930 --- /dev/null +++ b/src/Speckle.Core/SchemaVersioning/IObjectSchemaUpgrade.cs @@ -0,0 +1,51 @@ +using Speckle.Core.Models; + +namespace Speckle.Core.SchemaVersioning; + +public class TestBaseA : Base +{ + +} + +public class TestBaseB : Base +{ + +} + +public interface IObjectSchemaUpgrade where TInputType : class where TOutputType : class +{ + TOutputType Upgrade(TInputType incomingVersion); +} + +public class WallObjectUpgrader : IObjectSchemaUpgrade +{ + public TestBaseB Upgrade(TestBaseA input) + { + return new TestBaseB(); + } + + public Base Upgrade(Base incomingVersion) => Upgrade((TestBaseA) incomingVersion); +} + +public interface ISchemaObjectUpgrader + where TInputType : class where TOutputType : class +{ + +} + + +public class SchemaObjectUpgrader : ISchemaObjectUpgrader + where TInputType : class where TOutputType : class +{ + // look for stuff in assemblies like the type cache, this is somewhat similar + void BuildUpgrader() + { + // do we look for ISchemaObjectUpgrader + // do we look for a from and to value on some attribute? + // do have some abstract base that validates the input type matches, i.e. that the input that was as a Base + // matches the output + // should we consider linking this machinery to the typecache??? + } + + // question - could we just add typed/naming convention methods to the various classes? +} diff --git a/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs b/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs index 285171a..18239bb 100644 --- a/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs +++ b/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs @@ -43,12 +43,15 @@ public sealed class BaseObjectDeserializerV2 : ISpeckleDeserializer public int WorkerThreadCount { get; set; } = DefaultNumberThreads; private readonly ITypeCache _typeCache; + private readonly Version _payloadSchemaVersion; // POC: inject the TypeCacheManager, and interface out - public BaseObjectDeserializerV2(ITypeCache typeCache) + public BaseObjectDeserializerV2(ITypeCache typeCache, Version payloadSchemaVersion) { _typeCache = typeCache; _typeCache.EnsureCacheIsBuilt(); + + _payloadSchemaVersion = payloadSchemaVersion; } /// The JSON string of the object to be deserialized @@ -348,17 +351,13 @@ private Base Dict2Base(Dictionary dictObj) { string typeName = (string)dictObj[SerializationConstants.TYPE_DISCRIMINATOR]!; - // TODO: pass the version IN - Version schemaVersion = _typeCache.LoadedSchemaVersion; - // here we're getting the actual type to deserialise into, this won't be the type we return - CachedTypeInfo cachedTypeInfo = _typeCache.GetMatchedTypeOrLater(typeName, schemaVersion); + CachedTypeInfo cachedTypeInfo = _typeCache.GetMatchedTypeOrLater(typeName, _payloadSchemaVersion); Base baseObj = (Base) Activator.CreateInstance(cachedTypeInfo.Type); dictObj.Remove(SerializationConstants.TYPE_DISCRIMINATOR); dictObj.Remove(SerializationConstants.CLOSURE_PROPERTY_NAME); - dictObj.Remove(SerializationConstants.SCHEMA_VERSION); var props = cachedTypeInfo.Props; diff --git a/src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs b/src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs index 9039a4f..6edad15 100644 --- a/src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs +++ b/src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs @@ -10,8 +10,14 @@ namespace Speckle.Core.Serialisation.TypeCache; internal class VersionCache(string type) { public string Type { get; private set; } = type; - public List<(Version, CachedTypeInfo)> Versions => new (); + public List<(Version, CachedTypeInfo)> Versions { get; private set; } = new(); public CachedTypeInfo? LatestVersion; + + public void SortVersions() + { + // for some reason I can't get the tuple deconstructed (but it IS rather late) + Versions = Versions.OrderBy(v => v.Item1).ToList(); + } } public abstract class AbstractTypeCache : ITypeCache @@ -20,6 +26,8 @@ public abstract class AbstractTypeCache : ITypeCache private readonly Type _baseType; private readonly Dictionary _cachedTypes = new(); + private bool _cacheBuilt = false; + public Version LoadedSchemaVersion { get; private set; } private readonly string _namespacePrefix; @@ -46,6 +54,13 @@ public abstract class AbstractTypeCache : ITypeCache // could build in constructor.. but throwing from constructors generally frowned upon public void EnsureCacheIsBuilt() { + if (_cacheBuilt) + { + return; + } + + _cacheBuilt = true; + // POC: need a way to pick our own objects and not the DUI2 objects plus this is a touch weak :pain foreach (var assembly in _assemblies) { @@ -79,10 +94,26 @@ public void EnsureCacheIsBuilt() } } - // TODO: if we load versions but no latest, then this is an error // future incarnations may permit mutating the type - // i.e. decorate a new type with the name of the legacy name - + // so the return type is something other than base, they may also allow moving namespaces + // this is not currently possible because of the way the namespace has been used historically and how we are using it here + // we can probably tweak this to allow for namespace remapping - it would be nice if things began with Speckle :P + foreach (var typeVersions in _cachedTypes.Values) + { + if (typeVersions.LatestVersion == null) + { + // we cannot have non-matching types atm + // .i.e. a versioned Objects.Versions.V_1_2.0.Wall must have a corresponding Objects.Wall + // I imagine it would be possible to add some annotation somewhere to allow for this but now is not the time... + var versionNames = string.Join(",", typeVersions.Versions + .Select((v, typeCache) => v.Item1.ToString())).ToList(); + throw new ArgumentException( + $"The type {typeVersions.Type} has no latest - we have the following versions of this type: '{versionNames}'"); + } + + // sort the versions + typeVersions.SortVersions(); + } // TODO: order versioned cache by their version to save sorting every version look up diff --git a/src/Speckle.Core/Serialisation/TypeCache/CachedTypeInfo.cs b/src/Speckle.Core/Serialisation/TypeCache/CachedTypeInfo.cs index 31e4ffe..59a4996 100644 --- a/src/Speckle.Core/Serialisation/TypeCache/CachedTypeInfo.cs +++ b/src/Speckle.Core/Serialisation/TypeCache/CachedTypeInfo.cs @@ -5,18 +5,18 @@ namespace Speckle.Core.Serialisation.TypeCache; public sealed class CachedTypeInfo { - public string Key { get; private set; } + public string UnversionedTypeName { get; private set; } public Type Type { get; private set; } public ImmutableDictionary Props { get; private set; } public ImmutableList Callbacks { get; private set; } public CachedTypeInfo( - string key, + string unversionedTypeName, Type type, Dictionary props, List callbacks) { - Key = key; + UnversionedTypeName = unversionedTypeName; Type = type; Props = props.ToImmutableDictionary(); Callbacks = callbacks.ToImmutableList(); diff --git a/src/Speckle.Objects/Versions/V_0_1_0/Wall.cs b/src/Speckle.Objects/Versions/V_0_1_0/BuiltElements/Wall.cs similarity index 95% rename from src/Speckle.Objects/Versions/V_0_1_0/Wall.cs rename to src/Speckle.Objects/Versions/V_0_1_0/BuiltElements/Wall.cs index 469157b..0d11a74 100644 --- a/src/Speckle.Objects/Versions/V_0_1_0/Wall.cs +++ b/src/Speckle.Objects/Versions/V_0_1_0/BuiltElements/Wall.cs @@ -4,7 +4,7 @@ using Speckle.Core.Kits; using Speckle.Core.Models; -namespace Objects.Versions.V_0_1_0; +namespace Objects.Versions.V_0_1_0.BuiltElements; public class Wall : Base, IDisplayValue> { From bdeca89ec724fdc11d2c2edb591afdc18b26f83b Mon Sep 17 00:00:00 2001 From: Ian Hawley Date: Sat, 6 Jul 2024 23:13:35 +0100 Subject: [PATCH 09/18] It's compiling but there are some issues with the types --- src/Speckle.Core/Reflection/ITypeFinder.cs | 8 +++ .../Reflection/ITypeInstanceResolver.cs | 7 +++ .../Reflection/NamedTypeAttribute.cs | 24 +++++++++ src/Speckle.Core/Reflection/TypeFinder.cs | 26 ++++++++++ .../Reflection/TypeInstanceResolver.cs | 40 ++++++++++++++ .../AbstractSchemaObjectUpgrader.cs | 18 +++++++ .../SchemaVersioning/IObjectSchemaUpgrade.cs | 51 ------------------ .../ISchemaObjectUpgradeManager.cs | 7 +++ .../SchemaVersioning/ISchemaObjectUpgrader.cs | 10 ++++ .../SchemaObjectUpgradeManager.cs | 41 +++++++++++++++ .../TypeCache/AbstractTypeCache.cs | 52 ++++++++----------- src/Speckle.Objects/ObjectsTypeCache.cs | 9 +++- src/Speckle.Objects/Upgraders/WallUpgrader.cs | 19 +++++++ 13 files changed, 229 insertions(+), 83 deletions(-) create mode 100644 src/Speckle.Core/Reflection/ITypeFinder.cs create mode 100644 src/Speckle.Core/Reflection/ITypeInstanceResolver.cs create mode 100644 src/Speckle.Core/Reflection/NamedTypeAttribute.cs create mode 100644 src/Speckle.Core/Reflection/TypeFinder.cs create mode 100644 src/Speckle.Core/Reflection/TypeInstanceResolver.cs create mode 100644 src/Speckle.Core/SchemaVersioning/AbstractSchemaObjectUpgrader.cs delete mode 100644 src/Speckle.Core/SchemaVersioning/IObjectSchemaUpgrade.cs create mode 100644 src/Speckle.Core/SchemaVersioning/ISchemaObjectUpgradeManager.cs create mode 100644 src/Speckle.Core/SchemaVersioning/ISchemaObjectUpgrader.cs create mode 100644 src/Speckle.Core/SchemaVersioning/SchemaObjectUpgradeManager.cs create mode 100644 src/Speckle.Objects/Upgraders/WallUpgrader.cs diff --git a/src/Speckle.Core/Reflection/ITypeFinder.cs b/src/Speckle.Core/Reflection/ITypeFinder.cs new file mode 100644 index 0000000..38d26c0 --- /dev/null +++ b/src/Speckle.Core/Reflection/ITypeFinder.cs @@ -0,0 +1,8 @@ +using System.Reflection; + +namespace Speckle.Core.Reflection; + +public interface ITypeFinder +{ + IList GetTypesWhereSubclassOf(IEnumerable assemblies, Type subclassOf); +} diff --git a/src/Speckle.Core/Reflection/ITypeInstanceResolver.cs b/src/Speckle.Core/Reflection/ITypeInstanceResolver.cs new file mode 100644 index 0000000..834e83a --- /dev/null +++ b/src/Speckle.Core/Reflection/ITypeInstanceResolver.cs @@ -0,0 +1,7 @@ +namespace Speckle.Core.Reflection; + +public interface ITypeInstanceResolver where TType : class +{ + TType Resolve(string typeNameWithSuffix); + bool TryResolve(string typeNameWithSuffix, out TType instance); +} diff --git a/src/Speckle.Core/Reflection/NamedTypeAttribute.cs b/src/Speckle.Core/Reflection/NamedTypeAttribute.cs new file mode 100644 index 0000000..ff6f506 --- /dev/null +++ b/src/Speckle.Core/Reflection/NamedTypeAttribute.cs @@ -0,0 +1,24 @@ +using Speckle.Core.Common; + +namespace Speckle.Core.Reflection; + +[AttributeUsage(AttributeTargets.Class)] +public sealed class NamedTypeAttribute : Attribute +{ + public string TypeName { get; private set; } + public string TypeNameWithSuffix { get; private set; } + + public NamedTypeAttribute( + Type type, + string suffix) + { + TypeName = type.FullName.NotNull(); + TypeNameWithSuffix = $"TypeName{suffix}"; + } + + public NamedTypeAttribute( + string typeName) + { + TypeName = typeName; + } +} diff --git a/src/Speckle.Core/Reflection/TypeFinder.cs b/src/Speckle.Core/Reflection/TypeFinder.cs new file mode 100644 index 0000000..589ba44 --- /dev/null +++ b/src/Speckle.Core/Reflection/TypeFinder.cs @@ -0,0 +1,26 @@ +using System.Reflection; + +namespace Speckle.Core.Reflection; + +public class TypeFinder : ITypeFinder +{ + public IList GetTypesWhereSubclassOf(IEnumerable assemblies, Type subclassOf) + { + List types = new(); + + // this assumes the DUI2 objects are not already loaded + foreach (var assembly in assemblies) + { + try + { + types.AddRange(assembly.GetTypes().Where(x => x.IsSubclassOf(subclassOf) && !x.IsAbstract)); + } + catch (TypeLoadException) + { + // POC: guard against loading things that cause explosions + } + } + + return types; + } +} diff --git a/src/Speckle.Core/Reflection/TypeInstanceResolver.cs b/src/Speckle.Core/Reflection/TypeInstanceResolver.cs new file mode 100644 index 0000000..1a2560e --- /dev/null +++ b/src/Speckle.Core/Reflection/TypeInstanceResolver.cs @@ -0,0 +1,40 @@ +namespace Speckle.Core.Reflection; + +public class SingletonTypeInstanceResolver : ITypeInstanceResolver + where TType : class +{ + private readonly ITypeFinder _typeFinder; + private readonly Dictionary _typeInstances = new(); + + public SingletonTypeInstanceResolver(ITypeFinder typeFinder) + { + _typeFinder = typeFinder; + + // POC: not wild about evaluating this during construction but... is that still a thing? + // could be done on the fly... + var foundTypes = _typeFinder.GetTypesWhereSubclassOf(AppDomain.CurrentDomain.GetAssemblies(), typeof(TType)); + + // let's make an instance of each of these + // could also be done on the fly + foreach (var type in foundTypes) + { + // the type must have the attribute + // POC: we may want something other than default exception... + // maybe but there should always be one ONLY one + var namedType = (NamedTypeAttribute) type.GetCustomAttributes(typeof(NamedTypeAttribute), false).Single(); + _typeInstances[namedType.TypeNameWithSuffix] = (TType) Activator.CreateInstance(type); + } + } + + public TType Resolve(string typeNameWithSuffix) => _typeInstances[typeNameWithSuffix]; + + public bool TryResolve(string typeNameWithSuffix, out TType instance) + { + if (_typeInstances.TryGetValue(typeNameWithSuffix, out instance)) + { + return true; + } + + return false; + } +} diff --git a/src/Speckle.Core/SchemaVersioning/AbstractSchemaObjectUpgrader.cs b/src/Speckle.Core/SchemaVersioning/AbstractSchemaObjectUpgrader.cs new file mode 100644 index 0000000..473cf48 --- /dev/null +++ b/src/Speckle.Core/SchemaVersioning/AbstractSchemaObjectUpgrader.cs @@ -0,0 +1,18 @@ +using Speckle.Core.Models; + +namespace Speckle.Core.SchemaVersioning; + +public abstract class AbstractSchemaObjectBaseUpgrader : ISchemaObjectUpgrader + where TInputType : Base + where TOutputType : Base +{ + public Base Upgrade(Base input) + { + return Upgrade((TInputType) input); + } + + public abstract TOutputType Upgrade(TInputType input); + + public Version UpgradeFrom { get; protected set; } + public Version UpgradeTo { get; protected set; } +} diff --git a/src/Speckle.Core/SchemaVersioning/IObjectSchemaUpgrade.cs b/src/Speckle.Core/SchemaVersioning/IObjectSchemaUpgrade.cs deleted file mode 100644 index 7fdd930..0000000 --- a/src/Speckle.Core/SchemaVersioning/IObjectSchemaUpgrade.cs +++ /dev/null @@ -1,51 +0,0 @@ -using Speckle.Core.Models; - -namespace Speckle.Core.SchemaVersioning; - -public class TestBaseA : Base -{ - -} - -public class TestBaseB : Base -{ - -} - -public interface IObjectSchemaUpgrade where TInputType : class where TOutputType : class -{ - TOutputType Upgrade(TInputType incomingVersion); -} - -public class WallObjectUpgrader : IObjectSchemaUpgrade -{ - public TestBaseB Upgrade(TestBaseA input) - { - return new TestBaseB(); - } - - public Base Upgrade(Base incomingVersion) => Upgrade((TestBaseA) incomingVersion); -} - -public interface ISchemaObjectUpgrader - where TInputType : class where TOutputType : class -{ - -} - - -public class SchemaObjectUpgrader : ISchemaObjectUpgrader - where TInputType : class where TOutputType : class -{ - // look for stuff in assemblies like the type cache, this is somewhat similar - void BuildUpgrader() - { - // do we look for ISchemaObjectUpgrader - // do we look for a from and to value on some attribute? - // do have some abstract base that validates the input type matches, i.e. that the input that was as a Base - // matches the output - // should we consider linking this machinery to the typecache??? - } - - // question - could we just add typed/naming convention methods to the various classes? -} diff --git a/src/Speckle.Core/SchemaVersioning/ISchemaObjectUpgradeManager.cs b/src/Speckle.Core/SchemaVersioning/ISchemaObjectUpgradeManager.cs new file mode 100644 index 0000000..8aae25f --- /dev/null +++ b/src/Speckle.Core/SchemaVersioning/ISchemaObjectUpgradeManager.cs @@ -0,0 +1,7 @@ +namespace Speckle.Core.SchemaVersioning; + +public interface ISchemaObjectUpgradeManager + where TInputType : class where TOutputType : class +{ + +} diff --git a/src/Speckle.Core/SchemaVersioning/ISchemaObjectUpgrader.cs b/src/Speckle.Core/SchemaVersioning/ISchemaObjectUpgrader.cs new file mode 100644 index 0000000..3e2265d --- /dev/null +++ b/src/Speckle.Core/SchemaVersioning/ISchemaObjectUpgrader.cs @@ -0,0 +1,10 @@ +namespace Speckle.Core.SchemaVersioning; + +public interface ISchemaObjectUpgrader + where TInputType : class where TOutputType : class +{ + TOutputType Upgrade(TInputType input); + + Version UpgradeFrom { get; } + Version UpgradeTo { get; } +} diff --git a/src/Speckle.Core/SchemaVersioning/SchemaObjectUpgradeManager.cs b/src/Speckle.Core/SchemaVersioning/SchemaObjectUpgradeManager.cs new file mode 100644 index 0000000..6e5007f --- /dev/null +++ b/src/Speckle.Core/SchemaVersioning/SchemaObjectUpgradeManager.cs @@ -0,0 +1,41 @@ +using System.Reflection; +using Speckle.Core.Models; +using Speckle.Core.Reflection; + +namespace Speckle.Core.SchemaVersioning; + +public class SchemaObjectUpgradeManager : ISchemaObjectUpgradeManager + where TInputType : class where TOutputType : class +{ + // POC: use manual activation, newing the interface and caching the upgrader but... + // we will need to inject this later most likely + private readonly ITypeInstanceResolver> _typeInstanceResolver; + + public SchemaObjectUpgradeManager(ITypeInstanceResolver> typeInstanceResolver) + { + _typeInstanceResolver = typeInstanceResolver; + } + + public TOutputType UpgradeObject(TInputType input, string typeName, Version inputVersion, Version schemaVersion) + { + TOutputType upgraded; + + // we try and upgrade while-ever the versions don't match + while (inputVersion < schemaVersion) + { + // building this must be done consistently - maybe some helper? + string typeKey = $"{typeName}{inputVersion}"; + + if (!_typeInstanceResolver.TryResolve(typeKey, out ISchemaObjectUpgrader upgrader)) + { + // there's no upgrader for this + break; + } + + upgraded = upgrader.Upgrade(input); + inputVersion = upgrader.UpgradeTo; + } + + return null!;//input; + } +} diff --git a/src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs b/src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs index 6edad15..5616f43 100644 --- a/src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs +++ b/src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs @@ -4,6 +4,7 @@ using System.Runtime.Serialization; using Speckle.Core.Common; using Speckle.Core.Models; +using Speckle.Core.Reflection; namespace Speckle.Core.Serialisation.TypeCache; @@ -30,6 +31,7 @@ public abstract class AbstractTypeCache : ITypeCache public Version LoadedSchemaVersion { get; private set; } + private readonly ITypeFinder _typeFinder; private readonly string _namespacePrefix; private readonly string _versionNamespacePrefix; @@ -39,13 +41,18 @@ public abstract class AbstractTypeCache : ITypeCache IEnumerable assemblies, Type baseType, Version loadedSchemaVersion, - string namespacePrefix) + string namespacePrefix, + ITypeFinder typeFinder) { _assemblies = assemblies.ToList(); _baseType = baseType; LoadedSchemaVersion = loadedSchemaVersion; + + // FUTURE: we may need to associate the prefix with each assembly as the prefix may change + // better yet detach the namespaces & assemblies from the type naming _namespacePrefix = namespacePrefix; _versionNamespacePrefix = $"{_namespacePrefix}.Versions."; + _typeFinder = typeFinder; // POC: manually including core, not sure of the wisdom of this... _assemblies.Add(typeof(AbstractTypeCache).Assembly); @@ -60,44 +67,32 @@ public void EnsureCacheIsBuilt() } _cacheBuilt = true; - - // POC: need a way to pick our own objects and not the DUI2 objects plus this is a touch weak :pain - foreach (var assembly in _assemblies) + + foreach (Type type in _typeFinder.GetTypesWhereSubclassOf(_assemblies, _baseType)) { try { - var foundTypes = assembly.GetTypes().Where(x => x.IsSubclassOf(_baseType) && !x.IsAbstract); - - foreach (var type in foundTypes) - { - (string typeName, Version? version) = GetTypeNameAndVersion(type.FullName.NotNull()); - - try - { - var typeCacheInfo = new CachedTypeInfo( - typeName.NotNull(), - type, - GetPropertyInfo(type), - GetCallbacks(type)); - - CacheType(typeName, version, typeCacheInfo); - } - catch (TypeLoadException) - { - // POC: guard against loading things that cause explosions - } - } + (string typeName, Version? version) = GetTypeNameAndVersion(type.FullName.NotNull()); + + var typeCacheInfo = new CachedTypeInfo( + typeName.NotNull(), + type, + GetPropertyInfo(type), + GetCallbacks(type)); + + CacheType(typeName, version, typeCacheInfo); } catch (TypeLoadException) { // POC: guard against loading things that cause explosions - } + } } // future incarnations may permit mutating the type // so the return type is something other than base, they may also allow moving namespaces // this is not currently possible because of the way the namespace has been used historically and how we are using it here - // we can probably tweak this to allow for namespace remapping - it would be nice if things began with Speckle :P + // we can probably tweak this to allow for namespace remapping - it would be nice if things began with Speckle :) + // object version may also be able to retype things... foreach (var typeVersions in _cachedTypes.Values) { if (typeVersions.LatestVersion == null) @@ -114,9 +109,6 @@ public void EnsureCacheIsBuilt() // sort the versions typeVersions.SortVersions(); } - - // TODO: order versioned cache by their version to save sorting every version look up - FallbackType = new CachedTypeInfo( _baseType.FullName.NotNull(), diff --git a/src/Speckle.Objects/ObjectsTypeCache.cs b/src/Speckle.Objects/ObjectsTypeCache.cs index 445a7e5..48d031b 100644 --- a/src/Speckle.Objects/ObjectsTypeCache.cs +++ b/src/Speckle.Objects/ObjectsTypeCache.cs @@ -2,17 +2,22 @@ using System.Reflection; using Objects; using Speckle.Core.Models; +using Speckle.Core.Reflection; using Speckle.Core.Serialisation.TypeCache; namespace Speckle.Objects; public sealed class ObjectsTypeCache : AbstractTypeCache { - public ObjectsTypeCache(Version? version = null) + private readonly ITypeFinder _typeFinder; + + public ObjectsTypeCache(ITypeFinder typeFinder, Version? version = null) : base(new[] { typeof(ObjectsTypeCache).Assembly }, typeof(Base), version ?? SpeckleSchemaInfo.Version, - "Objects") + "Objects", + typeFinder) { + _typeFinder = typeFinder; } } diff --git a/src/Speckle.Objects/Upgraders/WallUpgrader.cs b/src/Speckle.Objects/Upgraders/WallUpgrader.cs new file mode 100644 index 0000000..0bdb059 --- /dev/null +++ b/src/Speckle.Objects/Upgraders/WallUpgrader.cs @@ -0,0 +1,19 @@ +using Objects.BuiltElements; +using Objects.BuiltElements.Revit; +using Speckle.Core.Reflection; +using Speckle.Core.SchemaVersioning; + +namespace Speckle.Objects.Upgraders; + +// POC: consider weakness of strings here +// the type passed in is a bit janky, if we want to move from say RevitWall to Wall, +// then we would need to have some care about where the type name came from, as in this example +// POC: the version could come from some constant tbh and then it won't be wrong... +[NamedType(typeof(Wall), "0.1.0")] +public sealed class WallUpgrader : AbstractSchemaObjectBaseUpgrader +{ + public override RevitWall Upgrade(Wall input) + { + return Upgrade((Wall)input); + } +} From fd00f366c78ce37d6353ee07efc8e6f137211c64 Mon Sep 17 00:00:00 2001 From: Ian Hawley Date: Sun, 7 Jul 2024 08:29:00 +0100 Subject: [PATCH 10/18] Can now find object upgraders --- .../Reflection/NamedTypeAttribute.cs | 17 ++++++------- .../Reflection/TypeInstanceResolver.cs | 10 +++++--- .../SchemaObjectUpgradeManager.cs | 25 ++++++++++++++++--- src/Speckle.Objects/Upgraders/WallUpgrader.cs | 9 ++++--- 4 files changed, 41 insertions(+), 20 deletions(-) diff --git a/src/Speckle.Core/Reflection/NamedTypeAttribute.cs b/src/Speckle.Core/Reflection/NamedTypeAttribute.cs index ff6f506..a0b92ed 100644 --- a/src/Speckle.Core/Reflection/NamedTypeAttribute.cs +++ b/src/Speckle.Core/Reflection/NamedTypeAttribute.cs @@ -6,19 +6,18 @@ namespace Speckle.Core.Reflection; public sealed class NamedTypeAttribute : Attribute { public string TypeName { get; private set; } - public string TypeNameWithSuffix { get; private set; } + public string TypeNameWithKeySuffix { get; private set; } public NamedTypeAttribute( Type type, - string suffix) + string keySuffix) { TypeName = type.FullName.NotNull(); - TypeNameWithSuffix = $"TypeName{suffix}"; - } - - public NamedTypeAttribute( - string typeName) - { - TypeName = typeName; + + // creating this key here needs to be consistent with where it is created + TypeNameWithKeySuffix = CreateTypeNameWithKeySuffix(TypeName, keySuffix); } + + // POC: we can absolutely decide how we wish to construct this + public static string CreateTypeNameWithKeySuffix(string typeName, string keySuffix) => $"{typeName}+{keySuffix}"; } diff --git a/src/Speckle.Core/Reflection/TypeInstanceResolver.cs b/src/Speckle.Core/Reflection/TypeInstanceResolver.cs index 1a2560e..d0b32e2 100644 --- a/src/Speckle.Core/Reflection/TypeInstanceResolver.cs +++ b/src/Speckle.Core/Reflection/TypeInstanceResolver.cs @@ -11,18 +11,20 @@ public SingletonTypeInstanceResolver(ITypeFinder typeFinder) _typeFinder = typeFinder; // POC: not wild about evaluating this during construction but... is that still a thing? - // could be done on the fly... + // could be done on the fly... it will require some locking semantics if used during deserialisation var foundTypes = _typeFinder.GetTypesWhereSubclassOf(AppDomain.CurrentDomain.GetAssemblies(), typeof(TType)); // let's make an instance of each of these // could also be done on the fly foreach (var type in foundTypes) { - // the type must have the attribute - // POC: we may want something other than default exception... + // the type must have the attribute so we know what we can upgrade *from* and upgrade *to* and + // also what the Name and suffix should be + + // POC: we may want something other than default exception when casting... // maybe but there should always be one ONLY one var namedType = (NamedTypeAttribute) type.GetCustomAttributes(typeof(NamedTypeAttribute), false).Single(); - _typeInstances[namedType.TypeNameWithSuffix] = (TType) Activator.CreateInstance(type); + _typeInstances[namedType.TypeNameWithKeySuffix] = (TType) Activator.CreateInstance(type); } } diff --git a/src/Speckle.Core/SchemaVersioning/SchemaObjectUpgradeManager.cs b/src/Speckle.Core/SchemaVersioning/SchemaObjectUpgradeManager.cs index 6e5007f..772978f 100644 --- a/src/Speckle.Core/SchemaVersioning/SchemaObjectUpgradeManager.cs +++ b/src/Speckle.Core/SchemaVersioning/SchemaObjectUpgradeManager.cs @@ -1,4 +1,5 @@ using System.Reflection; +using Speckle.Core.Logging; using Speckle.Core.Models; using Speckle.Core.Reflection; @@ -18,13 +19,13 @@ public SchemaObjectUpgradeManager(ITypeInstanceResolver upgrader)) { @@ -36,6 +37,22 @@ public TOutputType UpgradeObject(TInputType input, string typeName, Version inpu inputVersion = upgrader.UpgradeTo; } - return null!;//input; + // if we didn't do any upgrading, then we should be pass the input directly to the output + // BUT in cases where there is a conversion + if (upgraded is null) + { + upgraded = input as TOutputType; + if (upgraded is null) + { + // POC: we'll want some exception type here because we probably want this to explode always + // even if we change from Base to an IBase, it will be easy to do in a big bang, because Base can IBase cam implement + // I *think* this means we can neatly use this same class with different input and output types + // to nicely migrate to a different base if we wished to + throw new SpeckleException( + ($"Failed to convert '{input.GetType().FullName}' to '{typeof(TOutputType).FullName}'")); + } + } + + return upgraded; } } diff --git a/src/Speckle.Objects/Upgraders/WallUpgrader.cs b/src/Speckle.Objects/Upgraders/WallUpgrader.cs index 0bdb059..6af6c5a 100644 --- a/src/Speckle.Objects/Upgraders/WallUpgrader.cs +++ b/src/Speckle.Objects/Upgraders/WallUpgrader.cs @@ -10,10 +10,13 @@ namespace Speckle.Objects.Upgraders; // then we would need to have some care about where the type name came from, as in this example // POC: the version could come from some constant tbh and then it won't be wrong... [NamedType(typeof(Wall), "0.1.0")] -public sealed class WallUpgrader : AbstractSchemaObjectBaseUpgrader +public sealed class WallUpgrader : AbstractSchemaObjectBaseUpgrader { - public override RevitWall Upgrade(Wall input) + // edits should be done on the input, over creating copies for speed + public override RevitWall Upgrade(RevitWall input) { - return Upgrade((Wall)input); + + + return input; } } From 147343605024073d009e4cb2ec28033a8855875a Mon Sep 17 00:00:00 2001 From: Ian Hawley Date: Sun, 7 Jul 2024 08:53:45 +0100 Subject: [PATCH 11/18] Pulling the upgrade manager through all the way --- src/Speckle.Core/Api/Helpers.cs | 3 +++ .../Api/Operations/Operations.Receive.cs | 4 +++- .../Api/Operations/Operations.Serialize.cs | 5 +++-- .../ISchemaObjectUpgradeManager.cs | 2 +- .../Serialisation/BaseObjectDeserializerV2.cs | 16 ++++++++------ .../TypeCache/AbstractTypeCache.cs | 21 ++++++++----------- .../Serialisation/TypeCache/ITypeCache.cs | 2 +- 7 files changed, 30 insertions(+), 23 deletions(-) diff --git a/src/Speckle.Core/Api/Helpers.cs b/src/Speckle.Core/Api/Helpers.cs index e8c3a0f..63da26b 100644 --- a/src/Speckle.Core/Api/Helpers.cs +++ b/src/Speckle.Core/Api/Helpers.cs @@ -15,6 +15,7 @@ using Speckle.Core.Kits; using Speckle.Core.Logging; using Speckle.Core.Models; +using Speckle.Core.SchemaVersioning; using Speckle.Core.Serialisation.TypeCache; using Speckle.Core.Transports; using Speckle.Newtonsoft.Json; @@ -37,6 +38,7 @@ public static class Helpers public static async Task Receive( string stream, ITypeCache typeCache, + ISchemaObjectUpgradeManager objectUpgradeManager, System.Version schemaVersion, Account account = null, Action> onProgressAction = null, @@ -111,6 +113,7 @@ public static class Helpers .Receive( objectId, typeCache, + objectUpgradeManager, schemaVersion, transport, onProgressAction: onProgressAction, diff --git a/src/Speckle.Core/Api/Operations/Operations.Receive.cs b/src/Speckle.Core/Api/Operations/Operations.Receive.cs index fe2ca59..9e08de5 100644 --- a/src/Speckle.Core/Api/Operations/Operations.Receive.cs +++ b/src/Speckle.Core/Api/Operations/Operations.Receive.cs @@ -8,6 +8,7 @@ using Serilog.Context; using Speckle.Core.Logging; using Speckle.Core.Models; +using Speckle.Core.SchemaVersioning; using Speckle.Core.Serialisation; using Speckle.Core.Serialisation.TypeCache; using Speckle.Core.Transports; @@ -40,6 +41,7 @@ public static partial class Operations public static async Task Receive( string objectId, ITypeCache typeCache, + ISchemaObjectUpgradeManager objectUpgradeManager, System.Version schemaVersion, ITransport? remoteTransport = null, ITransport? localTransport = null, @@ -65,7 +67,7 @@ public static partial class Operations // Setup Serializer BaseObjectDeserializerV2 serializerV2 = - new(typeCache, schemaVersion) + new(typeCache, objectUpgradeManager, schemaVersion) { ReadTransport = localTransport, OnProgressAction = internalProgressAction, diff --git a/src/Speckle.Core/Api/Operations/Operations.Serialize.cs b/src/Speckle.Core/Api/Operations/Operations.Serialize.cs index 8ff415f..ef6fe27 100644 --- a/src/Speckle.Core/Api/Operations/Operations.Serialize.cs +++ b/src/Speckle.Core/Api/Operations/Operations.Serialize.cs @@ -1,5 +1,6 @@ using Speckle.Core.Logging; using Speckle.Core.Models; +using Speckle.Core.SchemaVersioning; using Speckle.Core.Serialisation; using Speckle.Core.Serialisation.TypeCache; using Speckle.Newtonsoft.Json; @@ -37,9 +38,9 @@ public static string Serialize(Base value, CancellationToken cancellationToken = /// was not valid JSON /// cannot be deserialised to type /// contains closure references (see Remarks) - public static Base Deserialize(string value, ITypeCache typeCache, System.Version payloadSchemaVersion, CancellationToken cancellationToken = default) + public static Base Deserialize(string value, ITypeCache typeCache, ISchemaObjectUpgradeManager objectUpgradeManager, System.Version payloadSchemaVersion, CancellationToken cancellationToken = default) { - var deserializer = new BaseObjectDeserializerV2(typeCache, payloadSchemaVersion) { CancellationToken = cancellationToken }; + var deserializer = new BaseObjectDeserializerV2(typeCache, objectUpgradeManager, payloadSchemaVersion) { CancellationToken = cancellationToken }; return deserializer.Deserialize(value); } } diff --git a/src/Speckle.Core/SchemaVersioning/ISchemaObjectUpgradeManager.cs b/src/Speckle.Core/SchemaVersioning/ISchemaObjectUpgradeManager.cs index 8aae25f..000240a 100644 --- a/src/Speckle.Core/SchemaVersioning/ISchemaObjectUpgradeManager.cs +++ b/src/Speckle.Core/SchemaVersioning/ISchemaObjectUpgradeManager.cs @@ -3,5 +3,5 @@ namespace Speckle.Core.SchemaVersioning; public interface ISchemaObjectUpgradeManager where TInputType : class where TOutputType : class { - + TOutputType UpgradeObject(TInputType input, string typeName, Version inputVersion, Version schemaVersion); } diff --git a/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs b/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs index 18239bb..e9279bc 100644 --- a/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs +++ b/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs @@ -9,6 +9,7 @@ using Speckle.Core.Kits; using Speckle.Core.Logging; using Speckle.Core.Models; +using Speckle.Core.SchemaVersioning; using Speckle.Core.Serialisation.SerializationUtilities; using Speckle.Core.Serialisation.TypeCache; using Speckle.Core.Transports; @@ -44,14 +45,16 @@ public sealed class BaseObjectDeserializerV2 : ISpeckleDeserializer private readonly ITypeCache _typeCache; private readonly Version _payloadSchemaVersion; + private readonly ISchemaObjectUpgradeManager _objectUpgradeManager; // POC: inject the TypeCacheManager, and interface out - public BaseObjectDeserializerV2(ITypeCache typeCache, Version payloadSchemaVersion) + public BaseObjectDeserializerV2(ITypeCache typeCache, ISchemaObjectUpgradeManager objectUpgradeManager, Version payloadSchemaVersion) { _typeCache = typeCache; - _typeCache.EnsureCacheIsBuilt(); - + _objectUpgradeManager = objectUpgradeManager; _payloadSchemaVersion = payloadSchemaVersion; + + _typeCache.EnsureCacheIsBuilt(); } /// The JSON string of the object to be deserialized @@ -349,10 +352,10 @@ private List<(string, int)> GetClosures(string rootObjectJson) private Base Dict2Base(Dictionary dictObj) { - string typeName = (string)dictObj[SerializationConstants.TYPE_DISCRIMINATOR]!; + string typeName = (string) dictObj[SerializationConstants.TYPE_DISCRIMINATOR]!; // here we're getting the actual type to deserialise into, this won't be the type we return - CachedTypeInfo cachedTypeInfo = _typeCache.GetMatchedTypeOrLater(typeName, _payloadSchemaVersion); + (Version objectVersion, CachedTypeInfo cachedTypeInfo) = _typeCache.GetMatchedTypeOrLater(typeName, _payloadSchemaVersion); Base baseObj = (Base) Activator.CreateInstance(cachedTypeInfo.Type); @@ -405,7 +408,8 @@ private Base Dict2Base(Dictionary dictObj) bb.filePath = bb.GetLocalDestinationPath(BlobStorageFolder); } - // TODO: perform any versioning we need to, to upgrade the type + // version the object + baseObj = _objectUpgradeManager.UpgradeObject(baseObj, cachedTypeInfo.Type.FullName, objectVersion, _payloadSchemaVersion); foreach (MethodInfo onDeserialized in onDeserializedCallbacks) { diff --git a/src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs b/src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs index 5616f43..3b5a9ec 100644 --- a/src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs +++ b/src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs @@ -1,13 +1,11 @@ -using System.Collections.Immutable; -using System.Collections.Specialized; -using System.Reflection; +using System.Reflection; using System.Runtime.Serialization; using Speckle.Core.Common; -using Speckle.Core.Models; using Speckle.Core.Reflection; namespace Speckle.Core.Serialisation.TypeCache; +// POC: could move internal class VersionCache(string type) { public string Type { get; private set; } = type; @@ -117,13 +115,12 @@ public void EnsureCacheIsBuilt() GetCallbacks(_baseType)); } - public CachedTypeInfo GetMatchedTypeOrLater(string speckleType, Version versionToMatch) + public (Version version, CachedTypeInfo cachedTypeInfo) GetMatchedTypeOrLater(string speckleType, Version versionToMatch) { int length = speckleType.Length; int end = length - 1; int start; - // _defaultCachedType should be created by now otherwise we should explode VersionCache? cachedVersions; do @@ -155,15 +152,15 @@ public CachedTypeInfo GetMatchedTypeOrLater(string speckleType, Version versionT } } while (start >= 0); - // why the hell is this moaning about it being null? - return FallbackType.NotNull(); + // FallbackType should exist + return (LoadedSchemaVersion, FallbackType.NotNull()); } - private CachedTypeInfo MatchOrLater(Version versionToMatch, VersionCache versionCache) + private (Version version, CachedTypeInfo cachedTypeInfo) MatchOrLater(Version versionToMatch, VersionCache versionCache) { if (versionToMatch == LoadedSchemaVersion) { - return versionCache.LatestVersion.NotNull(); + return (LoadedSchemaVersion, versionCache.LatestVersion.NotNull()); } // we could search or we can walk. We might be able to optomise this @@ -172,12 +169,12 @@ private CachedTypeInfo MatchOrLater(Version versionToMatch, VersionCache version // if it's a match or comes later, then use this if (cachedVersion.version >= versionToMatch) { - return cachedVersion.cachedTypeInfo; + return (cachedVersion.version, cachedVersion.cachedTypeInfo); } } // if we get here just use the latest version - return versionCache.LatestVersion.NotNull(); + return (LoadedSchemaVersion, versionCache.LatestVersion.NotNull()); } private void CacheType(string typeName, Version version, CachedTypeInfo typeCacheInfo) diff --git a/src/Speckle.Core/Serialisation/TypeCache/ITypeCache.cs b/src/Speckle.Core/Serialisation/TypeCache/ITypeCache.cs index 28a2303..1b02cd9 100644 --- a/src/Speckle.Core/Serialisation/TypeCache/ITypeCache.cs +++ b/src/Speckle.Core/Serialisation/TypeCache/ITypeCache.cs @@ -4,7 +4,7 @@ public interface ITypeCache { void EnsureCacheIsBuilt(); - CachedTypeInfo GetMatchedTypeOrLater(string speckleType, Version versionToMatch); + (Version version, CachedTypeInfo cachedTypeInfo) GetMatchedTypeOrLater(string speckleType, Version versionToMatch); Version LoadedSchemaVersion { get; } } From 9e95a8755ca9c6b771a21f124a1b3d3738c341e5 Mon Sep 17 00:00:00 2001 From: Ian Hawley Date: Sun, 7 Jul 2024 09:00:18 +0100 Subject: [PATCH 12/18] given the way I've built this, not sure if the source or destination is the key for the type Just need to test this now with something basic. --- src/Speckle.Objects/Upgraders/WallUpgrader.cs | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/Speckle.Objects/Upgraders/WallUpgrader.cs b/src/Speckle.Objects/Upgraders/WallUpgrader.cs index 6af6c5a..0864e9e 100644 --- a/src/Speckle.Objects/Upgraders/WallUpgrader.cs +++ b/src/Speckle.Objects/Upgraders/WallUpgrader.cs @@ -1,22 +1,30 @@ using Objects.BuiltElements; -using Objects.BuiltElements.Revit; using Speckle.Core.Reflection; using Speckle.Core.SchemaVersioning; +using SourceWall = Objects.Versions.V_0_1_0.BuiltElements.Wall; +using DestinationWall = Objects.BuiltElements.Wall; + namespace Speckle.Objects.Upgraders; // POC: consider weakness of strings here // the type passed in is a bit janky, if we want to move from say RevitWall to Wall, // then we would need to have some care about where the type name came from, as in this example // POC: the version could come from some constant tbh and then it won't be wrong... -[NamedType(typeof(Wall), "0.1.0")] -public sealed class WallUpgrader : AbstractSchemaObjectBaseUpgrader +// is the typename off the source or destination :pained face: +[NamedType(typeof(SourceWall), "0.1.0")] +public sealed class WallUpgrader : AbstractSchemaObjectBaseUpgrader { - // edits should be done on the input, over creating copies for speed - public override RevitWall Upgrade(RevitWall input) + public override DestinationWall Upgrade(SourceWall input) { + // we need to clone the source and make a new destination + + + + + // we need to fixup or add or otherwise - return input; + return new DestinationWall(); } } From 0c946a12c6d602dfe5e2227e5294ffa3c532d37b Mon Sep 17 00:00:00 2001 From: Ian Hawley Date: Sun, 7 Jul 2024 09:50:51 +0100 Subject: [PATCH 13/18] Added ridiculous Point upgrader, just to see it all working --- src/Speckle.Objects/Geometry/Point.cs | 91 ++++--- .../Versions/V_0_1_0/BuiltElements/Wall.cs | 44 ---- .../Versions/V_0_1_0/Geometry/Point.cs | 245 ++++++++++++++++++ .../V_0_1_0/Upgraders/PointUpgrader.cs} | 0 4 files changed, 294 insertions(+), 86 deletions(-) delete mode 100644 src/Speckle.Objects/Versions/V_0_1_0/BuiltElements/Wall.cs create mode 100644 src/Speckle.Objects/Versions/V_0_1_0/Geometry/Point.cs rename src/Speckle.Objects/{Upgraders/WallUpgrader.cs => Versions/V_0_1_0/Upgraders/PointUpgrader.cs} (100%) diff --git a/src/Speckle.Objects/Geometry/Point.cs b/src/Speckle.Objects/Geometry/Point.cs index 000ce69..2a3256a 100644 --- a/src/Speckle.Objects/Geometry/Point.cs +++ b/src/Speckle.Objects/Geometry/Point.cs @@ -1,12 +1,12 @@ -using System; -using System.Collections.Generic; +using Objects; +using Objects.Geometry; using Objects.Other; using Speckle.Core.Common; using Speckle.Core.Kits; using Speckle.Core.Models; using Speckle.Newtonsoft.Json; -namespace Objects.Geometry; +namespace Speckle.Objects.Geometry; /// /// A 3-dimensional point @@ -14,37 +14,38 @@ namespace Objects.Geometry; /// /// TODO: The Point class does not override the Equality operator, which means that there may be cases where `Equals` is used instead of `==`, as the comparison will be done by reference, not value. /// -public class Point : Base, ITransformable +public class Point_new : Base, ITransformable { /// - public Point() { } + public Point_new() { } /// - /// Constructs a new from a set of coordinates and it's units. + /// Constructs a new from a set of coordinates and it's units. /// /// The x coordinate /// The y coordinate /// The z coordinate /// The units of the point's coordinates. Defaults to Meters. /// The object's unique application ID - public Point(double x, double y, double z = 0d, string units = Units.Meters, string? applicationId = null) + public Point_new(double x, double y, double z = 0d, double w = 1d, string units = Units.Meters, string? applicationId = null) { this.x = x; this.y = y; this.z = z; + this.w = w; this.applicationId = applicationId; this.units = units; } /// - /// Constructs a new from a + /// Constructs a new from a /// /// The Vector whose coordinates will be used for the Point - public Point(Vector vector) - : this(vector.x, vector.y, vector.z, vector.units, vector.applicationId) { } + public Point_new(Vector vector) + : this(vector.x, vector.y, vector.z, 1d, vector.units, vector.applicationId) { } /// - /// Gets or sets the coordinates of the + /// Gets or sets the coordinates of the /// [JsonProperty(NullValueHandling = NullValueHandling.Ignore), Obsolete("Use x,y,z properties instead", true)] public List value @@ -72,9 +73,14 @@ public List value /// The z coordinate of the point. /// public double z { get; set; } + + /// + /// The w coordinate of the point. + /// + public double w { get; set; } /// - /// The units this is in. + /// The units this is in. /// This should be one of the units specified in /// public string units { get; set; } = Units.None; @@ -83,7 +89,7 @@ public List value public Box? bbox { get; set; } /// - public bool TransformTo(Transform transform, out Point transformed) + public bool TransformTo(Transform transform, out Point_new transformed) { var matrix = transform.matrix; @@ -93,20 +99,20 @@ public bool TransformTo(Transform transform, out Point transformed) var y = (this.x * matrix.M21 + this.y * matrix.M22 + this.z * matrix.M23 + unitFactor * matrix.M24) / divisor; var z = (this.x * matrix.M31 + this.y * matrix.M32 + this.z * matrix.M33 + unitFactor * matrix.M34) / divisor; - transformed = new Point(x, y, z) { units = units, applicationId = applicationId }; + transformed = new Point_new(x, y, z) { units = units, applicationId = applicationId }; return true; } /// public bool TransformTo(Transform transform, out ITransformable transformed) { - var res = TransformTo(transform, out Point pt); + var res = TransformTo(transform, out Point_new pt); transformed = pt; return res; } /// - /// Returns the coordinates of this as a list of numbers + /// Returns the coordinates of this as a list of numbers /// /// A list of coordinates {x, y, z} public List ToList() @@ -115,18 +121,18 @@ public List ToList() } /// - /// Creates a new based on a list of coordinates and the unit they're drawn in. + /// Creates a new based on a list of coordinates and the unit they're drawn in. /// /// The list of coordinates {x, y, z} /// The units the coordinates are in - /// A new with the provided coordinates. - public static Point FromList(IList list, string units) + /// A new with the provided coordinates. + public static Point_new FromList(IList list, string units) { - return new Point(list[0], list[1], list[2], units); + return new Point_new(list[0], list[1], list[2], 1d, units); } /// - /// Deconstructs a into it's coordinates and units + /// Deconstructs a into it's coordinates and units /// /// The x coordinate /// The y coordinate @@ -139,7 +145,7 @@ public void Deconstruct(out double x, out double y, out double z, out string? un } /// - /// Deconstructs a into it's coordinates and units + /// Deconstructs a into it's coordinates and units /// /// The x coordinate /// The y coordinate @@ -151,22 +157,22 @@ public void Deconstruct(out double x, out double y, out double z) z = this.z; } - public static Point operator +(Point point1, Point point2) => - new(point1.x + point2.x, point1.y + point2.y, point1.z + point2.z, point1.units); + public static Point_new operator +(Point_new point1, Point_new point2) => + new(point1.x + point2.x, point1.y + point2.y, point1.z + point2.z, 1d, point1.units); - public static Point operator -(Point point1, Point point2) => - new(point1.x - point2.x, point1.y - point2.y, point1.z - point2.z, point1.units); + public static Point_new operator -(Point_new point1, Point_new point2) => + new(point1.x - point2.x, point1.y - point2.y, point1.z - point2.z, 1d, point1.units); - public static Point operator *(Point point1, Point point2) => - new(point1.x * point2.x, point1.y * point2.y, point1.z * point2.z, point1.units); + public static Point_new operator *(Point_new point1, Point_new point2) => + new(point1.x * point2.x, point1.y * point2.y, point1.z * point2.z, 1d, point1.units); - public static Point operator *(Point point, double val) => - new(point.x * val, point.y * val, point.z * val, point.units); + public static Point_new operator *(Point_new pointNew, double val) => + new(pointNew.x * val, pointNew.y * val, pointNew.z * val, 1d, pointNew.units); - public static Point operator /(Point point, double val) => - new(point.x / val, point.y / val, point.z / val, point.units); + public static Point_new operator /(Point_new pointNew, double val) => + new(pointNew.x / val, pointNew.y / val, pointNew.z / val, 1d, pointNew.units); - public static bool operator ==(Point? point1, Point? point2) + public static bool operator ==(Point_new? point1, Point_new? point2) { if (point1 is null && point2 is null) { @@ -180,7 +186,7 @@ public void Deconstruct(out double x, out double y, out double z) return point1.units == point2.units && point1.x == point2.x && point1.y == point2.y && point1.z == point2.z; } - public static bool operator !=(Point? point1, Point? point2) => !(point1 == point2); + public static bool operator !=(Point_new? point1, Point_new? point2) => !(point1 == point2); /// /// Computes a point equidistant from two points. @@ -188,12 +194,13 @@ public void Deconstruct(out double x, out double y, out double z) /// First point. /// Second point. /// A point at the same distance from and - public static Point Midpoint(Point point1, Point point2) + public static Point_new Midpoint(Point_new point1, Point_new point2) { - return new Point( + return new Point_new( 0.5 * (point1.x + point2.x), 0.5 * (point1.y + point2.y), 0.5 * (point1.z + point2.z), + 1d, point1.units ); } @@ -204,7 +211,7 @@ public static Point Midpoint(Point point1, Point point2) /// First point. /// Second point. /// The distance from to - public static double Distance(Point point1, Point point2) + public static double Distance(Point_new point1, Point_new point2) { return Math.Sqrt( Math.Pow(point1.x - point2.x, 2) + Math.Pow(point1.y - point2.y, 2) + Math.Pow(point1.z - point2.z, 2) @@ -214,14 +221,14 @@ public static double Distance(Point point1, Point point2) /// /// Computes the distance between two points. /// - /// point for distance measurement + /// point for distance measurement /// The length of the line between this and the other point - public double DistanceTo(Point point) + public double DistanceTo(Point_new pointNew) { - return Math.Sqrt(Math.Pow(x - point.x, 2) + Math.Pow(y - point.y, 2) + Math.Pow(z - point.z, 2)); + return Math.Sqrt(Math.Pow(x - pointNew.x, 2) + Math.Pow(y - pointNew.y, 2) + Math.Pow(z - pointNew.z, 2)); } - public static Point Add(Point left, Point right) + public static Point_new Add(Point_new left, Point_new right) { throw new NotImplementedException(); } @@ -238,7 +245,7 @@ public override bool Equals(object obj) return false; } - return this == (Point)obj; + return this == (Point_new)obj; } public override int GetHashCode() => HashCode.Of(units).And(x).And(y).And(y); diff --git a/src/Speckle.Objects/Versions/V_0_1_0/BuiltElements/Wall.cs b/src/Speckle.Objects/Versions/V_0_1_0/BuiltElements/Wall.cs deleted file mode 100644 index 0d11a74..0000000 --- a/src/Speckle.Objects/Versions/V_0_1_0/BuiltElements/Wall.cs +++ /dev/null @@ -1,44 +0,0 @@ -using Objects; -using Objects.BuiltElements; -using Objects.Geometry; -using Speckle.Core.Kits; -using Speckle.Core.Models; - -namespace Objects.Versions.V_0_1_0.BuiltElements; - -public class Wall : Base, IDisplayValue> -{ - public Wall() { } - - /// - /// SchemaBuilder constructor for a Speckle wall - /// - /// - /// - /// - /// Assign units when using this constructor due to param - [SchemaInfo("Wall", "Creates a Speckle wall", "BIM", "Architecture")] - public Wall( - double height, - [SchemaMainParam] ICurve baseLine, - [SchemaParamInfo("Any nested elements that this wall might have")] List? elements = null - ) - { - this.height = height; - this.baseLine = baseLine; - this.elements = elements; - } - - public double height { get; set; } - - [DetachProperty] - public List? elements { get; set; } - - public ICurve baseLine { get; set; } - public virtual Level? level { get; internal set; } - - public string units { get; set; } - - [DetachProperty] - public List displayValue { get; set; } -} diff --git a/src/Speckle.Objects/Versions/V_0_1_0/Geometry/Point.cs b/src/Speckle.Objects/Versions/V_0_1_0/Geometry/Point.cs new file mode 100644 index 0000000..000ce69 --- /dev/null +++ b/src/Speckle.Objects/Versions/V_0_1_0/Geometry/Point.cs @@ -0,0 +1,245 @@ +using System; +using System.Collections.Generic; +using Objects.Other; +using Speckle.Core.Common; +using Speckle.Core.Kits; +using Speckle.Core.Models; +using Speckle.Newtonsoft.Json; + +namespace Objects.Geometry; + +/// +/// A 3-dimensional point +/// +/// +/// TODO: The Point class does not override the Equality operator, which means that there may be cases where `Equals` is used instead of `==`, as the comparison will be done by reference, not value. +/// +public class Point : Base, ITransformable +{ + /// + public Point() { } + + /// + /// Constructs a new from a set of coordinates and it's units. + /// + /// The x coordinate + /// The y coordinate + /// The z coordinate + /// The units of the point's coordinates. Defaults to Meters. + /// The object's unique application ID + public Point(double x, double y, double z = 0d, string units = Units.Meters, string? applicationId = null) + { + this.x = x; + this.y = y; + this.z = z; + this.applicationId = applicationId; + this.units = units; + } + + /// + /// Constructs a new from a + /// + /// The Vector whose coordinates will be used for the Point + public Point(Vector vector) + : this(vector.x, vector.y, vector.z, vector.units, vector.applicationId) { } + + /// + /// Gets or sets the coordinates of the + /// + [JsonProperty(NullValueHandling = NullValueHandling.Ignore), Obsolete("Use x,y,z properties instead", true)] + public List value + { + get => null!; + set + { + x = value[0]; + y = value[1]; + z = value.Count > 2 ? value[2] : 0; + } + } + + /// + /// The x coordinate of the point. + /// + public double x { get; set; } + + /// + /// The y coordinate of the point. + /// + public double y { get; set; } + + /// + /// The z coordinate of the point. + /// + public double z { get; set; } + + /// + /// The units this is in. + /// This should be one of the units specified in + /// + public string units { get; set; } = Units.None; + + [JsonIgnore, Obsolete("Bounding box no longer applicable to point as of 2.18", true)] + public Box? bbox { get; set; } + + /// + public bool TransformTo(Transform transform, out Point transformed) + { + var matrix = transform.matrix; + + var unitFactor = Units.GetConversionFactor(transform.units, units); // applied to translation vector + var divisor = matrix.M41 + matrix.M42 + matrix.M43 + unitFactor * matrix.M44; + var x = (this.x * matrix.M11 + this.y * matrix.M12 + this.z * matrix.M13 + unitFactor * matrix.M14) / divisor; + var y = (this.x * matrix.M21 + this.y * matrix.M22 + this.z * matrix.M23 + unitFactor * matrix.M24) / divisor; + var z = (this.x * matrix.M31 + this.y * matrix.M32 + this.z * matrix.M33 + unitFactor * matrix.M34) / divisor; + + transformed = new Point(x, y, z) { units = units, applicationId = applicationId }; + return true; + } + + /// + public bool TransformTo(Transform transform, out ITransformable transformed) + { + var res = TransformTo(transform, out Point pt); + transformed = pt; + return res; + } + + /// + /// Returns the coordinates of this as a list of numbers + /// + /// A list of coordinates {x, y, z} + public List ToList() + { + return new List { x, y, z }; + } + + /// + /// Creates a new based on a list of coordinates and the unit they're drawn in. + /// + /// The list of coordinates {x, y, z} + /// The units the coordinates are in + /// A new with the provided coordinates. + public static Point FromList(IList list, string units) + { + return new Point(list[0], list[1], list[2], units); + } + + /// + /// Deconstructs a into it's coordinates and units + /// + /// The x coordinate + /// The y coordinate + /// The z coordinate + /// The units the point's coordinates are in. + public void Deconstruct(out double x, out double y, out double z, out string? units) + { + Deconstruct(out x, out y, out z); + units = this.units; + } + + /// + /// Deconstructs a into it's coordinates and units + /// + /// The x coordinate + /// The y coordinate + /// The z coordinate + public void Deconstruct(out double x, out double y, out double z) + { + x = this.x; + y = this.y; + z = this.z; + } + + public static Point operator +(Point point1, Point point2) => + new(point1.x + point2.x, point1.y + point2.y, point1.z + point2.z, point1.units); + + public static Point operator -(Point point1, Point point2) => + new(point1.x - point2.x, point1.y - point2.y, point1.z - point2.z, point1.units); + + public static Point operator *(Point point1, Point point2) => + new(point1.x * point2.x, point1.y * point2.y, point1.z * point2.z, point1.units); + + public static Point operator *(Point point, double val) => + new(point.x * val, point.y * val, point.z * val, point.units); + + public static Point operator /(Point point, double val) => + new(point.x / val, point.y / val, point.z / val, point.units); + + public static bool operator ==(Point? point1, Point? point2) + { + if (point1 is null && point2 is null) + { + return true; + } + else if (point1 is null || point2 is null) + { + return false; + } + + return point1.units == point2.units && point1.x == point2.x && point1.y == point2.y && point1.z == point2.z; + } + + public static bool operator !=(Point? point1, Point? point2) => !(point1 == point2); + + /// + /// Computes a point equidistant from two points. + /// + /// First point. + /// Second point. + /// A point at the same distance from and + public static Point Midpoint(Point point1, Point point2) + { + return new Point( + 0.5 * (point1.x + point2.x), + 0.5 * (point1.y + point2.y), + 0.5 * (point1.z + point2.z), + point1.units + ); + } + + /// + /// Computes the distance between two points + /// + /// First point. + /// Second point. + /// The distance from to + public static double Distance(Point point1, Point point2) + { + return Math.Sqrt( + Math.Pow(point1.x - point2.x, 2) + Math.Pow(point1.y - point2.y, 2) + Math.Pow(point1.z - point2.z, 2) + ); + } + + /// + /// Computes the distance between two points. + /// + /// point for distance measurement + /// The length of the line between this and the other point + public double DistanceTo(Point point) + { + return Math.Sqrt(Math.Pow(x - point.x, 2) + Math.Pow(y - point.y, 2) + Math.Pow(z - point.z, 2)); + } + + public static Point Add(Point left, Point right) + { + throw new NotImplementedException(); + } + + public override bool Equals(object obj) + { + if (ReferenceEquals(this, obj)) + { + return true; + } + + if (ReferenceEquals(obj, null)) + { + return false; + } + + return this == (Point)obj; + } + + public override int GetHashCode() => HashCode.Of(units).And(x).And(y).And(y); +} diff --git a/src/Speckle.Objects/Upgraders/WallUpgrader.cs b/src/Speckle.Objects/Versions/V_0_1_0/Upgraders/PointUpgrader.cs similarity index 100% rename from src/Speckle.Objects/Upgraders/WallUpgrader.cs rename to src/Speckle.Objects/Versions/V_0_1_0/Upgraders/PointUpgrader.cs From 5d9a4029203e2233182873e1750cf60f20141936 Mon Sep 17 00:00:00 2001 From: Ian Hawley Date: Sun, 7 Jul 2024 09:57:01 +0100 Subject: [PATCH 14/18] Actually there was more for previous commit 'Added ridiculous Point upgrader, just to see it all working' Also tidied some unused usings --- src/Speckle.Core/Api/GraphQL/Models.cs | 2 + src/Speckle.Core/Kits/Attributes.cs | 2 - src/Speckle.Core/Kits/Exceptions.cs | 1 - src/Speckle.Core/Kits/Units.cs | 2 - ...er.cs => SingletonTypeInstanceResolver.cs} | 0 .../AbstractSchemaObjectUpgrader.cs | 10 ++- .../Serialisation/BaseObjectDeserializerV2.cs | 4 +- src/Speckle.Objects/Geometry/Point.cs | 83 +++++++++---------- src/Speckle.Objects/Speckle.Objects.csproj | 4 + .../Versions/V_0_1_0/Geometry/Point.cs | 6 +- .../V_0_1_0/Upgraders/PointUpgrader.cs | 26 +++--- 11 files changed, 71 insertions(+), 69 deletions(-) rename src/Speckle.Core/Reflection/{TypeInstanceResolver.cs => SingletonTypeInstanceResolver.cs} (100%) diff --git a/src/Speckle.Core/Api/GraphQL/Models.cs b/src/Speckle.Core/Api/GraphQL/Models.cs index c659d82..87cd51b 100644 --- a/src/Speckle.Core/Api/GraphQL/Models.cs +++ b/src/Speckle.Core/Api/GraphQL/Models.cs @@ -75,6 +75,8 @@ public class CommitCreateInput public string sourceApplication { get; set; } = ".net"; public int totalChildrenCount { get; set; } public List parents { get; set; } + + public string SchemaVersion { get; set; } [Obsolete("Please use the parents property. This property will be removed in later versions")] public List previousCommitIds { get; set; } diff --git a/src/Speckle.Core/Kits/Attributes.cs b/src/Speckle.Core/Kits/Attributes.cs index f706859..b46a6e1 100644 --- a/src/Speckle.Core/Kits/Attributes.cs +++ b/src/Speckle.Core/Kits/Attributes.cs @@ -1,6 +1,4 @@ #nullable disable -using System; - namespace Speckle.Core.Kits; [AttributeUsage(AttributeTargets.Constructor)] diff --git a/src/Speckle.Core/Kits/Exceptions.cs b/src/Speckle.Core/Kits/Exceptions.cs index b269ce3..98c81c9 100644 --- a/src/Speckle.Core/Kits/Exceptions.cs +++ b/src/Speckle.Core/Kits/Exceptions.cs @@ -1,4 +1,3 @@ -using System; using Speckle.Core.Logging; namespace Speckle.Core.Kits; diff --git a/src/Speckle.Core/Kits/Units.cs b/src/Speckle.Core/Kits/Units.cs index e3fec56..b939b53 100644 --- a/src/Speckle.Core/Kits/Units.cs +++ b/src/Speckle.Core/Kits/Units.cs @@ -1,5 +1,3 @@ -using System; -using System.Collections.Generic; using System.Diagnostics.Contracts; using Speckle.Core.Common; diff --git a/src/Speckle.Core/Reflection/TypeInstanceResolver.cs b/src/Speckle.Core/Reflection/SingletonTypeInstanceResolver.cs similarity index 100% rename from src/Speckle.Core/Reflection/TypeInstanceResolver.cs rename to src/Speckle.Core/Reflection/SingletonTypeInstanceResolver.cs diff --git a/src/Speckle.Core/SchemaVersioning/AbstractSchemaObjectUpgrader.cs b/src/Speckle.Core/SchemaVersioning/AbstractSchemaObjectUpgrader.cs index 473cf48..48344c5 100644 --- a/src/Speckle.Core/SchemaVersioning/AbstractSchemaObjectUpgrader.cs +++ b/src/Speckle.Core/SchemaVersioning/AbstractSchemaObjectUpgrader.cs @@ -6,6 +6,12 @@ public abstract class AbstractSchemaObjectBaseUpgrader where TInputType : Base where TOutputType : Base { + public AbstractSchemaObjectBaseUpgrader(Version from, Version to) + { + UpgradeFrom = from; + UpgradeTo = to; + } + public Base Upgrade(Base input) { return Upgrade((TInputType) input); @@ -13,6 +19,6 @@ public Base Upgrade(Base input) public abstract TOutputType Upgrade(TInputType input); - public Version UpgradeFrom { get; protected set; } - public Version UpgradeTo { get; protected set; } + public Version UpgradeFrom { get; private set; } + public Version UpgradeTo { get; private set; } } diff --git a/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs b/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs index e9279bc..c3bc487 100644 --- a/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs +++ b/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs @@ -409,8 +409,10 @@ private Base Dict2Base(Dictionary dictObj) } // version the object - baseObj = _objectUpgradeManager.UpgradeObject(baseObj, cachedTypeInfo.Type.FullName, objectVersion, _payloadSchemaVersion); + baseObj = _objectUpgradeManager.UpgradeObject(baseObj, cachedTypeInfo.Type.FullName.NotNull(), objectVersion, _payloadSchemaVersion); + // POC: what are these callback methods? are they used? + // Do they need calling on the object BEFORE upgrading it? Do they need calling AGAIN after upgrading it? foreach (MethodInfo onDeserialized in onDeserializedCallbacks) { onDeserialized.Invoke(baseObj, new object?[] { null }); diff --git a/src/Speckle.Objects/Geometry/Point.cs b/src/Speckle.Objects/Geometry/Point.cs index 2a3256a..1fcf171 100644 --- a/src/Speckle.Objects/Geometry/Point.cs +++ b/src/Speckle.Objects/Geometry/Point.cs @@ -1,12 +1,10 @@ -using Objects; -using Objects.Geometry; using Objects.Other; using Speckle.Core.Common; using Speckle.Core.Kits; using Speckle.Core.Models; using Speckle.Newtonsoft.Json; -namespace Speckle.Objects.Geometry; +namespace Objects.Geometry; /// /// A 3-dimensional point @@ -14,20 +12,20 @@ namespace Speckle.Objects.Geometry; /// /// TODO: The Point class does not override the Equality operator, which means that there may be cases where `Equals` is used instead of `==`, as the comparison will be done by reference, not value. /// -public class Point_new : Base, ITransformable +public class Point : Base, ITransformable { /// - public Point_new() { } + public Point() { } /// - /// Constructs a new from a set of coordinates and it's units. + /// Constructs a new from a set of coordinates and it's units. /// /// The x coordinate /// The y coordinate /// The z coordinate /// The units of the point's coordinates. Defaults to Meters. /// The object's unique application ID - public Point_new(double x, double y, double z = 0d, double w = 1d, string units = Units.Meters, string? applicationId = null) + public Point(double x, double y, double z = 0d, string units = Units.Meters, string? applicationId = null, double w = 1d) { this.x = x; this.y = y; @@ -38,14 +36,14 @@ public Point_new(double x, double y, double z = 0d, double w = 1d, string units } /// - /// Constructs a new from a + /// Constructs a new from a /// /// The Vector whose coordinates will be used for the Point - public Point_new(Vector vector) - : this(vector.x, vector.y, vector.z, 1d, vector.units, vector.applicationId) { } + public Point(Vector vector) + : this(vector.x, vector.y, vector.z, vector.units, vector.applicationId) { } /// - /// Gets or sets the coordinates of the + /// Gets or sets the coordinates of the /// [JsonProperty(NullValueHandling = NullValueHandling.Ignore), Obsolete("Use x,y,z properties instead", true)] public List value @@ -80,7 +78,7 @@ public List value public double w { get; set; } /// - /// The units this is in. + /// The units this is in. /// This should be one of the units specified in /// public string units { get; set; } = Units.None; @@ -89,7 +87,7 @@ public List value public Box? bbox { get; set; } /// - public bool TransformTo(Transform transform, out Point_new transformed) + public bool TransformTo(Transform transform, out Point transformed) { var matrix = transform.matrix; @@ -99,20 +97,20 @@ public bool TransformTo(Transform transform, out Point_new transformed) var y = (this.x * matrix.M21 + this.y * matrix.M22 + this.z * matrix.M23 + unitFactor * matrix.M24) / divisor; var z = (this.x * matrix.M31 + this.y * matrix.M32 + this.z * matrix.M33 + unitFactor * matrix.M34) / divisor; - transformed = new Point_new(x, y, z) { units = units, applicationId = applicationId }; + transformed = new Point(x, y, z) { units = units, applicationId = applicationId }; return true; } /// public bool TransformTo(Transform transform, out ITransformable transformed) { - var res = TransformTo(transform, out Point_new pt); + var res = TransformTo(transform, out Point pt); transformed = pt; return res; } /// - /// Returns the coordinates of this as a list of numbers + /// Returns the coordinates of this as a list of numbers /// /// A list of coordinates {x, y, z} public List ToList() @@ -121,18 +119,18 @@ public List ToList() } /// - /// Creates a new based on a list of coordinates and the unit they're drawn in. + /// Creates a new based on a list of coordinates and the unit they're drawn in. /// /// The list of coordinates {x, y, z} /// The units the coordinates are in - /// A new with the provided coordinates. - public static Point_new FromList(IList list, string units) + /// A new with the provided coordinates. + public static Point FromList(IList list, string units) { - return new Point_new(list[0], list[1], list[2], 1d, units); + return new Point(list[0], list[1], list[2], units); } /// - /// Deconstructs a into it's coordinates and units + /// Deconstructs a into it's coordinates and units /// /// The x coordinate /// The y coordinate @@ -145,7 +143,7 @@ public void Deconstruct(out double x, out double y, out double z, out string? un } /// - /// Deconstructs a into it's coordinates and units + /// Deconstructs a into it's coordinates and units /// /// The x coordinate /// The y coordinate @@ -157,22 +155,22 @@ public void Deconstruct(out double x, out double y, out double z) z = this.z; } - public static Point_new operator +(Point_new point1, Point_new point2) => - new(point1.x + point2.x, point1.y + point2.y, point1.z + point2.z, 1d, point1.units); + public static Point operator +(Point point1, Point point2) => + new(point1.x + point2.x, point1.y + point2.y, point1.z + point2.z, point1.units); - public static Point_new operator -(Point_new point1, Point_new point2) => - new(point1.x - point2.x, point1.y - point2.y, point1.z - point2.z, 1d, point1.units); + public static Point operator -(Point point1, Point point2) => + new(point1.x - point2.x, point1.y - point2.y, point1.z - point2.z, point1.units); - public static Point_new operator *(Point_new point1, Point_new point2) => - new(point1.x * point2.x, point1.y * point2.y, point1.z * point2.z, 1d, point1.units); + public static Point operator *(Point point1, Point point2) => + new(point1.x * point2.x, point1.y * point2.y, point1.z * point2.z, point1.units); - public static Point_new operator *(Point_new pointNew, double val) => - new(pointNew.x * val, pointNew.y * val, pointNew.z * val, 1d, pointNew.units); + public static Point operator *(Point point, double val) => + new(point.x * val, point.y * val, point.z * val, point.units); - public static Point_new operator /(Point_new pointNew, double val) => - new(pointNew.x / val, pointNew.y / val, pointNew.z / val, 1d, pointNew.units); + public static Point operator /(Point point, double val) => + new(point.x / val, point.y / val, point.z / val, point.units); - public static bool operator ==(Point_new? point1, Point_new? point2) + public static bool operator ==(Point? point1, Point? point2) { if (point1 is null && point2 is null) { @@ -186,7 +184,7 @@ public void Deconstruct(out double x, out double y, out double z) return point1.units == point2.units && point1.x == point2.x && point1.y == point2.y && point1.z == point2.z; } - public static bool operator !=(Point_new? point1, Point_new? point2) => !(point1 == point2); + public static bool operator !=(Point? point1, Point? point2) => !(point1 == point2); /// /// Computes a point equidistant from two points. @@ -194,13 +192,12 @@ public void Deconstruct(out double x, out double y, out double z) /// First point. /// Second point. /// A point at the same distance from and - public static Point_new Midpoint(Point_new point1, Point_new point2) + public static Point Midpoint(Point point1, Point point2) { - return new Point_new( + return new Point( 0.5 * (point1.x + point2.x), 0.5 * (point1.y + point2.y), 0.5 * (point1.z + point2.z), - 1d, point1.units ); } @@ -211,7 +208,7 @@ public static Point_new Midpoint(Point_new point1, Point_new point2) /// First point. /// Second point. /// The distance from to - public static double Distance(Point_new point1, Point_new point2) + public static double Distance(Point point1, Point point2) { return Math.Sqrt( Math.Pow(point1.x - point2.x, 2) + Math.Pow(point1.y - point2.y, 2) + Math.Pow(point1.z - point2.z, 2) @@ -221,14 +218,14 @@ public static double Distance(Point_new point1, Point_new point2) /// /// Computes the distance between two points. /// - /// point for distance measurement + /// point for distance measurement /// The length of the line between this and the other point - public double DistanceTo(Point_new pointNew) + public double DistanceTo(Point point) { - return Math.Sqrt(Math.Pow(x - pointNew.x, 2) + Math.Pow(y - pointNew.y, 2) + Math.Pow(z - pointNew.z, 2)); + return Math.Sqrt(Math.Pow(x - point.x, 2) + Math.Pow(y - point.y, 2) + Math.Pow(z - point.z, 2)); } - public static Point_new Add(Point_new left, Point_new right) + public static Point Add(Point left, Point right) { throw new NotImplementedException(); } @@ -245,7 +242,7 @@ public override bool Equals(object obj) return false; } - return this == (Point_new)obj; + return this == (Point)obj; } public override int GetHashCode() => HashCode.Of(units).And(x).And(y).And(y); diff --git a/src/Speckle.Objects/Speckle.Objects.csproj b/src/Speckle.Objects/Speckle.Objects.csproj index e4ff519..5be4cf9 100644 --- a/src/Speckle.Objects/Speckle.Objects.csproj +++ b/src/Speckle.Objects/Speckle.Objects.csproj @@ -17,4 +17,8 @@ + + + + diff --git a/src/Speckle.Objects/Versions/V_0_1_0/Geometry/Point.cs b/src/Speckle.Objects/Versions/V_0_1_0/Geometry/Point.cs index 000ce69..7d2d0d2 100644 --- a/src/Speckle.Objects/Versions/V_0_1_0/Geometry/Point.cs +++ b/src/Speckle.Objects/Versions/V_0_1_0/Geometry/Point.cs @@ -1,12 +1,12 @@ -using System; -using System.Collections.Generic; +using Objects; +using Objects.Geometry; using Objects.Other; using Speckle.Core.Common; using Speckle.Core.Kits; using Speckle.Core.Models; using Speckle.Newtonsoft.Json; -namespace Objects.Geometry; +namespace Objects.Versions.V_0_1_0.Geometry; /// /// A 3-dimensional point diff --git a/src/Speckle.Objects/Versions/V_0_1_0/Upgraders/PointUpgrader.cs b/src/Speckle.Objects/Versions/V_0_1_0/Upgraders/PointUpgrader.cs index 0864e9e..b743691 100644 --- a/src/Speckle.Objects/Versions/V_0_1_0/Upgraders/PointUpgrader.cs +++ b/src/Speckle.Objects/Versions/V_0_1_0/Upgraders/PointUpgrader.cs @@ -1,30 +1,26 @@ -using Objects.BuiltElements; using Speckle.Core.Reflection; using Speckle.Core.SchemaVersioning; -using SourceWall = Objects.Versions.V_0_1_0.BuiltElements.Wall; -using DestinationWall = Objects.BuiltElements.Wall; +using Source = Objects.Versions.V_0_1_0.Geometry.Point; +using Destination = Objects.Geometry.Point; -namespace Speckle.Objects.Upgraders; +namespace Speckle.Objects.Versions.V_0_1_0.Upgraders; // POC: consider weakness of strings here // the type passed in is a bit janky, if we want to move from say RevitWall to Wall, // then we would need to have some care about where the type name came from, as in this example // POC: the version could come from some constant tbh and then it won't be wrong... // is the typename off the source or destination :pained face: -[NamedType(typeof(SourceWall), "0.1.0")] -public sealed class WallUpgrader : AbstractSchemaObjectBaseUpgrader +[NamedType(typeof(Source), "0.1.0")] +public sealed class PointUpgrader : AbstractSchemaObjectBaseUpgrader { - public override DestinationWall Upgrade(SourceWall input) + public PointUpgrader() : base(new Version(0,1,0), new Version(0,2, 0)) { - // we need to clone the source and make a new destination - - - - // we need to fixup or add or otherwise - - - return new DestinationWall(); + } + + public override Destination Upgrade(Source input) + { + return new Destination(input.x, input.y, input.z); } } From ca18da058567b14287d74ee48af3875830151aa8 Mon Sep 17 00:00:00 2001 From: Ian Hawley Date: Sun, 7 Jul 2024 11:03:16 +0100 Subject: [PATCH 15/18] versioning a point appears to work.... :D --- src/Speckle.Core/Reflection/ITypeFinder.cs | 1 + .../SingletonTypeInstanceResolver.cs | 2 +- src/Speckle.Core/Reflection/TypeFinder.cs | 26 ++++++++++++++-- .../ISchemaObjectUpgradeManager.cs | 2 +- .../SchemaObjectUpgradeManager.cs | 11 ++++--- .../Serialisation/BaseObjectDeserializerV2.cs | 30 +++++++++++-------- .../TypeCache/AbstractTypeCache.cs | 3 +- src/Speckle.Core/SpeckleObjectSchema.cs | 2 +- src/Speckle.Objects/Speckle.Objects.csproj | 2 +- src/Speckle.Objects/SpeckleSchemaInfo.cs | 2 +- .../{V_0_1_0 => V_0_0_0}/Geometry/Point.cs | 2 +- .../Upgraders/PointUpgrader.cs | 8 ++--- 12 files changed, 60 insertions(+), 31 deletions(-) rename src/Speckle.Objects/Versions/{V_0_1_0 => V_0_0_0}/Geometry/Point.cs (99%) rename src/Speckle.Objects/Versions/{V_0_1_0 => V_0_0_0}/Upgraders/PointUpgrader.cs (76%) diff --git a/src/Speckle.Core/Reflection/ITypeFinder.cs b/src/Speckle.Core/Reflection/ITypeFinder.cs index 38d26c0..53ad935 100644 --- a/src/Speckle.Core/Reflection/ITypeFinder.cs +++ b/src/Speckle.Core/Reflection/ITypeFinder.cs @@ -5,4 +5,5 @@ namespace Speckle.Core.Reflection; public interface ITypeFinder { IList GetTypesWhereSubclassOf(IEnumerable assemblies, Type subclassOf); + IList GetTypesWhereImplementing(IEnumerable assemblies, Type subclassOf); } diff --git a/src/Speckle.Core/Reflection/SingletonTypeInstanceResolver.cs b/src/Speckle.Core/Reflection/SingletonTypeInstanceResolver.cs index d0b32e2..276b344 100644 --- a/src/Speckle.Core/Reflection/SingletonTypeInstanceResolver.cs +++ b/src/Speckle.Core/Reflection/SingletonTypeInstanceResolver.cs @@ -12,7 +12,7 @@ public SingletonTypeInstanceResolver(ITypeFinder typeFinder) // POC: not wild about evaluating this during construction but... is that still a thing? // could be done on the fly... it will require some locking semantics if used during deserialisation - var foundTypes = _typeFinder.GetTypesWhereSubclassOf(AppDomain.CurrentDomain.GetAssemblies(), typeof(TType)); + var foundTypes = _typeFinder.GetTypesWhereImplementing(AppDomain.CurrentDomain.GetAssemblies(), typeof(TType)); // let's make an instance of each of these // could also be done on the fly diff --git a/src/Speckle.Core/Reflection/TypeFinder.cs b/src/Speckle.Core/Reflection/TypeFinder.cs index 589ba44..0279f3b 100644 --- a/src/Speckle.Core/Reflection/TypeFinder.cs +++ b/src/Speckle.Core/Reflection/TypeFinder.cs @@ -15,9 +15,31 @@ public IList GetTypesWhereSubclassOf(IEnumerable assemblies, Typ { types.AddRange(assembly.GetTypes().Where(x => x.IsSubclassOf(subclassOf) && !x.IsAbstract)); } - catch (TypeLoadException) + // POC: right one? more? + catch (ReflectionTypeLoadException) { - // POC: guard against loading things that cause explosions + // POC: guard against loading things that cause explosions due to not being able to load assemblies but are not real issues + } + } + + return types; + } + + public IList GetTypesWhereImplementing(IEnumerable assemblies, Type subclassOf) + { + List types = new(); + + // this assumes the DUI2 objects are not already loaded + foreach (var assembly in assemblies) + { + try + { + types.AddRange(assembly.GetTypes().Where(x => x.GetInterfaces().Contains(subclassOf) && !x.IsAbstract)); + } + // POC: right one? more? + catch (ReflectionTypeLoadException) + { + // POC: guard against loading things that cause explosions due to not being able to load assemblies but are not real issues } } diff --git a/src/Speckle.Core/SchemaVersioning/ISchemaObjectUpgradeManager.cs b/src/Speckle.Core/SchemaVersioning/ISchemaObjectUpgradeManager.cs index 000240a..8b05634 100644 --- a/src/Speckle.Core/SchemaVersioning/ISchemaObjectUpgradeManager.cs +++ b/src/Speckle.Core/SchemaVersioning/ISchemaObjectUpgradeManager.cs @@ -3,5 +3,5 @@ namespace Speckle.Core.SchemaVersioning; public interface ISchemaObjectUpgradeManager where TInputType : class where TOutputType : class { - TOutputType UpgradeObject(TInputType input, string typeName, Version inputVersion, Version schemaVersion); + TOutputType UpgradeObject(TInputType input, string typeName, Version inputSchemaVersion, Version loadedSchemaVersion); } diff --git a/src/Speckle.Core/SchemaVersioning/SchemaObjectUpgradeManager.cs b/src/Speckle.Core/SchemaVersioning/SchemaObjectUpgradeManager.cs index 772978f..1b38b67 100644 --- a/src/Speckle.Core/SchemaVersioning/SchemaObjectUpgradeManager.cs +++ b/src/Speckle.Core/SchemaVersioning/SchemaObjectUpgradeManager.cs @@ -1,6 +1,4 @@ -using System.Reflection; using Speckle.Core.Logging; -using Speckle.Core.Models; using Speckle.Core.Reflection; namespace Speckle.Core.SchemaVersioning; @@ -17,16 +15,17 @@ public SchemaObjectUpgradeManager(ITypeInstanceResolver upgrader)) { // there's no upgrader for this @@ -34,7 +33,7 @@ public TOutputType UpgradeObject(TInputType input, string typeName, Version inpu } upgraded = upgrader.Upgrade(input); - inputVersion = upgrader.UpgradeTo; + inputSchemaVersion = upgrader.UpgradeTo; } // if we didn't do any upgrading, then we should be pass the input directly to the output diff --git a/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs b/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs index c3bc487..ea1809e 100644 --- a/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs +++ b/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs @@ -1,12 +1,6 @@ -using System; -using System.Collections.Generic; using System.Diagnostics; -using System.IO; using System.Reflection; -using System.Threading; -using System.Threading.Tasks; using Speckle.Core.Common; -using Speckle.Core.Kits; using Speckle.Core.Logging; using Speckle.Core.Models; using Speckle.Core.SchemaVersioning; @@ -355,17 +349,17 @@ private Base Dict2Base(Dictionary dictObj) string typeName = (string) dictObj[SerializationConstants.TYPE_DISCRIMINATOR]!; // here we're getting the actual type to deserialise into, this won't be the type we return + // POC: is there any guarantee this can't happen? probably not ATM + // it's almost certainly a bug if the type returned is a version and not the latest! (Version objectVersion, CachedTypeInfo cachedTypeInfo) = _typeCache.GetMatchedTypeOrLater(typeName, _payloadSchemaVersion); Base baseObj = (Base) Activator.CreateInstance(cachedTypeInfo.Type); - - dictObj.Remove(SerializationConstants.TYPE_DISCRIMINATOR); - dictObj.Remove(SerializationConstants.CLOSURE_PROPERTY_NAME); - var props = cachedTypeInfo.Props; - var onDeserializedCallbacks = cachedTypeInfo.Callbacks; + dictObj.Remove(SerializationConstants.TYPE_DISCRIMINATOR); + dictObj.Remove(SerializationConstants.CLOSURE_PROPERTY_NAME); + foreach (var entry in dictObj) { string lowerPropertyName = entry.Key.ToLower(); @@ -407,9 +401,20 @@ private Base Dict2Base(Dictionary dictObj) { bb.filePath = bb.GetLocalDestinationPath(BlobStorageFolder); } + + + // if (cachedTypeInfo.Type.FullName.NotNull().ToLower().Contains("point")) + // { + // Debug.WriteLine("breakpoint :p"); + // } // version the object - baseObj = _objectUpgradeManager.UpgradeObject(baseObj, cachedTypeInfo.Type.FullName.NotNull(), objectVersion, _payloadSchemaVersion); + // POC: we need to cache the right name here, because reflecting to get the mame is meh + baseObj = _objectUpgradeManager.UpgradeObject( + baseObj, + cachedTypeInfo.Type.FullName.NotNull(), + _payloadSchemaVersion, + _typeCache.LoadedSchemaVersion); // POC: what are these callback methods? are they used? // Do they need calling on the object BEFORE upgrading it? Do they need calling AGAIN after upgrading it? @@ -418,6 +423,7 @@ private Base Dict2Base(Dictionary dictObj) onDeserialized.Invoke(baseObj, new object?[] { null }); } + // POC: we need to be returning the latest version return baseObj; } } diff --git a/src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs b/src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs index 3b5a9ec..e8319a9 100644 --- a/src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs +++ b/src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs @@ -80,7 +80,8 @@ public void EnsureCacheIsBuilt() CacheType(typeName, version, typeCacheInfo); } - catch (TypeLoadException) + // POC: right one? more? + catch (ReflectionTypeLoadException) { // POC: guard against loading things that cause explosions } diff --git a/src/Speckle.Core/SpeckleObjectSchema.cs b/src/Speckle.Core/SpeckleObjectSchema.cs index e3bcb4c..dc27c15 100644 --- a/src/Speckle.Core/SpeckleObjectSchema.cs +++ b/src/Speckle.Core/SpeckleObjectSchema.cs @@ -9,5 +9,5 @@ namespace Speckle.Core; public class SpeckleObjectSchema { // POC: I'm not sure about using the Version object ATM, strings may be just as a straight-forward... - public static readonly Version Version = new Version(3, 0, 0); + public static readonly Version Version = new Version(0, 2, 0); } diff --git a/src/Speckle.Objects/Speckle.Objects.csproj b/src/Speckle.Objects/Speckle.Objects.csproj index 5be4cf9..f81e2d8 100644 --- a/src/Speckle.Objects/Speckle.Objects.csproj +++ b/src/Speckle.Objects/Speckle.Objects.csproj @@ -19,6 +19,6 @@ - + diff --git a/src/Speckle.Objects/SpeckleSchemaInfo.cs b/src/Speckle.Objects/SpeckleSchemaInfo.cs index ed930ac..6d2c828 100644 --- a/src/Speckle.Objects/SpeckleSchemaInfo.cs +++ b/src/Speckle.Objects/SpeckleSchemaInfo.cs @@ -2,5 +2,5 @@ public static class SpeckleSchemaInfo { - public static readonly Version Version = new(0, 2, 0); + public static readonly Version Version = new(0, 1, 0); } diff --git a/src/Speckle.Objects/Versions/V_0_1_0/Geometry/Point.cs b/src/Speckle.Objects/Versions/V_0_0_0/Geometry/Point.cs similarity index 99% rename from src/Speckle.Objects/Versions/V_0_1_0/Geometry/Point.cs rename to src/Speckle.Objects/Versions/V_0_0_0/Geometry/Point.cs index 7d2d0d2..fb7eb4f 100644 --- a/src/Speckle.Objects/Versions/V_0_1_0/Geometry/Point.cs +++ b/src/Speckle.Objects/Versions/V_0_0_0/Geometry/Point.cs @@ -6,7 +6,7 @@ using Speckle.Core.Models; using Speckle.Newtonsoft.Json; -namespace Objects.Versions.V_0_1_0.Geometry; +namespace Objects.Versions.V_0_0_0.Geometry; /// /// A 3-dimensional point diff --git a/src/Speckle.Objects/Versions/V_0_1_0/Upgraders/PointUpgrader.cs b/src/Speckle.Objects/Versions/V_0_0_0/Upgraders/PointUpgrader.cs similarity index 76% rename from src/Speckle.Objects/Versions/V_0_1_0/Upgraders/PointUpgrader.cs rename to src/Speckle.Objects/Versions/V_0_0_0/Upgraders/PointUpgrader.cs index b743691..67bcefb 100644 --- a/src/Speckle.Objects/Versions/V_0_1_0/Upgraders/PointUpgrader.cs +++ b/src/Speckle.Objects/Versions/V_0_0_0/Upgraders/PointUpgrader.cs @@ -1,20 +1,20 @@ using Speckle.Core.Reflection; using Speckle.Core.SchemaVersioning; -using Source = Objects.Versions.V_0_1_0.Geometry.Point; +using Source = Objects.Versions.V_0_0_0.Geometry.Point; using Destination = Objects.Geometry.Point; -namespace Speckle.Objects.Versions.V_0_1_0.Upgraders; +namespace Speckle.Objects.Versions.V_0_0_0.Upgraders; // POC: consider weakness of strings here // the type passed in is a bit janky, if we want to move from say RevitWall to Wall, // then we would need to have some care about where the type name came from, as in this example // POC: the version could come from some constant tbh and then it won't be wrong... // is the typename off the source or destination :pained face: -[NamedType(typeof(Source), "0.1.0")] +[NamedType(typeof(Source), "0.0.0")] public sealed class PointUpgrader : AbstractSchemaObjectBaseUpgrader { - public PointUpgrader() : base(new Version(0,1,0), new Version(0,2, 0)) + public PointUpgrader() : base(new Version(0,0,0), new Version(0,1, 0)) { } From 31f12d00ca5eaf2b209f2e041dddc9bfa37d7aed Mon Sep 17 00:00:00 2001 From: Ian Hawley Date: Sat, 13 Jul 2024 15:25:38 +0100 Subject: [PATCH 16/18] Considering repointing SDK --- src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs | 6 ------ src/Speckle.Core/Speckle.Core.csproj | 1 + src/Speckle.Objects/SpeckleSchemaInfo.cs | 2 +- src/Speckle.Objects/Versions/V_0_0_0/Geometry/Point.cs | 1 - 4 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs b/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs index ea1809e..e47d353 100644 --- a/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs +++ b/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs @@ -401,12 +401,6 @@ private Base Dict2Base(Dictionary dictObj) { bb.filePath = bb.GetLocalDestinationPath(BlobStorageFolder); } - - - // if (cachedTypeInfo.Type.FullName.NotNull().ToLower().Contains("point")) - // { - // Debug.WriteLine("breakpoint :p"); - // } // version the object // POC: we need to cache the right name here, because reflecting to get the mame is meh diff --git a/src/Speckle.Core/Speckle.Core.csproj b/src/Speckle.Core/Speckle.Core.csproj index 0c499dc..d7c9055 100644 --- a/src/Speckle.Core/Speckle.Core.csproj +++ b/src/Speckle.Core/Speckle.Core.csproj @@ -10,6 +10,7 @@ $(PackageTags) core true System.Runtime.CompilerServices.IsExternalInit;System.Runtime.CompilerServices.RequiresLocationAttribute + true diff --git a/src/Speckle.Objects/SpeckleSchemaInfo.cs b/src/Speckle.Objects/SpeckleSchemaInfo.cs index 6d2c828..ed930ac 100644 --- a/src/Speckle.Objects/SpeckleSchemaInfo.cs +++ b/src/Speckle.Objects/SpeckleSchemaInfo.cs @@ -2,5 +2,5 @@ public static class SpeckleSchemaInfo { - public static readonly Version Version = new(0, 1, 0); + public static readonly Version Version = new(0, 2, 0); } diff --git a/src/Speckle.Objects/Versions/V_0_0_0/Geometry/Point.cs b/src/Speckle.Objects/Versions/V_0_0_0/Geometry/Point.cs index fb7eb4f..c0868ef 100644 --- a/src/Speckle.Objects/Versions/V_0_0_0/Geometry/Point.cs +++ b/src/Speckle.Objects/Versions/V_0_0_0/Geometry/Point.cs @@ -1,4 +1,3 @@ -using Objects; using Objects.Geometry; using Objects.Other; using Speckle.Core.Common; From 6168b3973af5ad940e8de63423234db93be79375 Mon Sep 17 00:00:00 2001 From: Ian Hawley Date: Sun, 14 Jul 2024 15:08:58 +0100 Subject: [PATCH 17/18] removed dead merge code SHOULD REMOVE "generate package" --- .../Serialisation/BaseObjectDeserializerV2.cs | 11 ----------- src/Speckle.Objects/Speckle.Objects.csproj | 5 +++-- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs b/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs index 9500d09..19a5e8b 100644 --- a/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs +++ b/src/Speckle.Core/Serialisation/BaseObjectDeserializerV2.cs @@ -356,17 +356,6 @@ private Base Dict2Base(Dictionary dictObj) Base baseObj = (Base) Activator.CreateInstance(cachedTypeInfo.Type); var props = cachedTypeInfo.Props; var onDeserializedCallbacks = cachedTypeInfo.Callbacks; -// ======= -// string typeName = (string)dictObj[TYPE_DISCRIMINATOR].NotNull(); -// Type type = BaseObjectSerializationUtilities.GetType(typeName); -// Base baseObj = (Base)Activator.CreateInstance(type); -// -// dictObj.Remove(TYPE_DISCRIMINATOR); -// dictObj.Remove("__closure"); -// -// Dictionary staticProperties = BaseObjectSerializationUtilities.GetTypeProperties(typeName); -// List onDeserializedCallbacks = BaseObjectSerializationUtilities.GetOnDeserializedCallbacks(typeName); -// >>>>>>> dev dictObj.Remove(SerializationConstants.TYPE_DISCRIMINATOR); dictObj.Remove(SerializationConstants.CLOSURE_PROPERTY_NAME); diff --git a/src/Speckle.Objects/Speckle.Objects.csproj b/src/Speckle.Objects/Speckle.Objects.csproj index f81e2d8..55b2773 100644 --- a/src/Speckle.Objects/Speckle.Objects.csproj +++ b/src/Speckle.Objects/Speckle.Objects.csproj @@ -8,6 +8,7 @@ Objects is the default object model for Speckle $(PackageTags), objects System.Runtime.CompilerServices.IsExternalInit;System.Runtime.CompilerServices.RequiresLocationAttribute + true @@ -15,10 +16,10 @@ - + - + From 8fb6c104eed21c85ce13417ce0b7376b54a86944 Mon Sep 17 00:00:00 2001 From: Ian Hawley Date: Wed, 17 Jul 2024 06:56:41 +0200 Subject: [PATCH 18/18] minor fixes, most notably to where the Schema version for the commit we are deserialising comes from --- src/Speckle.Core/Api/GraphQL/Models/Version.cs | 5 +++++ src/Speckle.Core/Api/Operations/Operations.Receive.cs | 4 ++-- src/Speckle.Core/Serialisation/SerializationConstants.cs | 1 - .../Serialisation/TypeCache/AbstractTypeCache.cs | 9 +++++++-- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/Speckle.Core/Api/GraphQL/Models/Version.cs b/src/Speckle.Core/Api/GraphQL/Models/Version.cs index 1aa46b0..0ebadc7 100644 --- a/src/Speckle.Core/Api/GraphQL/Models/Version.cs +++ b/src/Speckle.Core/Api/GraphQL/Models/Version.cs @@ -15,4 +15,9 @@ public sealed class Version public Uri previewUrl { get; init; } public string referencedObject { get; init; } public string sourceApplication { get; init; } + + public System.Version SchemaVersion { get; init; } + + // POC: is this the right place for a const? + public const string EARLIEST_SCHEMA_VERSION_STRING = "0.0.0"; } diff --git a/src/Speckle.Core/Api/Operations/Operations.Receive.cs b/src/Speckle.Core/Api/Operations/Operations.Receive.cs index 9e08de5..bdfa183 100644 --- a/src/Speckle.Core/Api/Operations/Operations.Receive.cs +++ b/src/Speckle.Core/Api/Operations/Operations.Receive.cs @@ -42,7 +42,7 @@ public static partial class Operations string objectId, ITypeCache typeCache, ISchemaObjectUpgradeManager objectUpgradeManager, - System.Version schemaVersion, + System.Version payloadSchemaVersion, ITransport? remoteTransport = null, ITransport? localTransport = null, Action>? onProgressAction = null, @@ -67,7 +67,7 @@ public static partial class Operations // Setup Serializer BaseObjectDeserializerV2 serializerV2 = - new(typeCache, objectUpgradeManager, schemaVersion) + new(typeCache, objectUpgradeManager, payloadSchemaVersion) { ReadTransport = localTransport, OnProgressAction = internalProgressAction, diff --git a/src/Speckle.Core/Serialisation/SerializationConstants.cs b/src/Speckle.Core/Serialisation/SerializationConstants.cs index 07e6b2c..c8a134d 100644 --- a/src/Speckle.Core/Serialisation/SerializationConstants.cs +++ b/src/Speckle.Core/Serialisation/SerializationConstants.cs @@ -6,5 +6,4 @@ public class SerializationConstants { public const string TYPE_DISCRIMINATOR = nameof(Base.speckle_type); public const string CLOSURE_PROPERTY_NAME = "__closure"; - public const string SCHEMA_VERSION = "__schema_version"; } diff --git a/src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs b/src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs index e8319a9..af39332 100644 --- a/src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs +++ b/src/Speckle.Core/Serialisation/TypeCache/AbstractTypeCache.cs @@ -6,12 +6,17 @@ namespace Speckle.Core.Serialisation.TypeCache; // POC: could move -internal class VersionCache(string type) +internal class VersionCache { - public string Type { get; private set; } = type; + public string Type { get; private set; } public List<(Version, CachedTypeInfo)> Versions { get; private set; } = new(); public CachedTypeInfo? LatestVersion; + public VersionCache(string type) + { + Type = type; + } + public void SortVersions() { // for some reason I can't get the tuple deconstructed (but it IS rather late)