diff --git a/src/Tableau.Migration/Engine/Manifest/MigrationManifestSerializer.cs b/src/Tableau.Migration/Engine/Manifest/MigrationManifestSerializer.cs new file mode 100644 index 00000000..b9027237 --- /dev/null +++ b/src/Tableau.Migration/Engine/Manifest/MigrationManifestSerializer.cs @@ -0,0 +1,158 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Collections.Immutable; +using System.IO; +using System.IO.Abstractions; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Tableau.Migration.JsonConverters; +using Tableau.Migration.JsonConverters.SerializableObjects; +using Tableau.Migration.Resources; + + +namespace Tableau.Migration.Engine.Manifest +{ + /// + /// Provides functionality to serialize and deserialize migration manifests in JSON format. + /// + public class MigrationManifestSerializer + { + private readonly IFileSystem _fileSystem; + private readonly ISharedResourcesLocalizer _localizer; + private readonly ILoggerFactory _loggerFactory; + + private readonly ImmutableArray _converters; + + /// + /// Initializes a new instance of the class. + /// + public MigrationManifestSerializer(IFileSystem fileSystem, ISharedResourcesLocalizer localizer, ILoggerFactory loggerFactory) + { + _fileSystem = fileSystem; + _localizer = localizer; + _loggerFactory = loggerFactory; + + _converters = CreateConverters(); + } + + /// + /// This is the current MigrationManifest.ManifestVersion that this serializer supports. + /// + public const uint SupportedManifestVersion = MigrationManifest.LatestManifestVersion; + + + /// + /// Creates the list of JSON converters used by the MigrationManifestSerializer. + /// + /// This is a static method so the tests can use the same list converters. + /// An immutable array of JSON converters. + internal static ImmutableArray CreateConverters() + { + return new JsonConverter[] + { + new PythonExceptionConverter(), + new SerializedExceptionJsonConverter(), + new BuildResponseExceptionJsonConverter(), + new JobJsonConverter(), + new TimeoutJobExceptionJsonConverter(), + new RestExceptionJsonConverter(), + new FailedJobExceptionJsonConverter(), + new ExceptionJsonConverterFactory(), // This needs to be at the end. This list is ordered. + }.ToImmutableArray(); + } + + private JsonSerializerOptions MergeJsonOptions(JsonSerializerOptions? jsonOptions) + { + jsonOptions ??= new() { WriteIndented = true }; + + foreach (var converter in _converters) + { + jsonOptions.Converters.Add(converter); + } + + return jsonOptions; + } + + /// + /// Saves a manifest in JSON format. + /// + /// This async function does not take a cancellation token. This is because the saving should happen, + /// no matter what the status of the cancellation token is. Otherwise the manifest is not saved if the migration is cancelled. + /// The manifest to save. + /// The file path to save the manifest to. + /// Optional JSON options to use. + public async Task SaveAsync(IMigrationManifest manifest, string path, JsonSerializerOptions? jsonOptions = null) + { + jsonOptions = MergeJsonOptions(jsonOptions); + + var dir = Path.GetDirectoryName(path); + if (dir is not null && !_fileSystem.Directory.Exists(dir)) + { + _fileSystem.Directory.CreateDirectory(dir); + } + + var serializableManifest = new SerializableMigrationManifest(manifest); + + var file = _fileSystem.File.Create(path); + await using (file.ConfigureAwait(false)) + { + // If cancellation was requested, we still need to save the file, so use the default token. + await JsonSerializer.SerializeAsync(file, serializableManifest, jsonOptions, default) + .ConfigureAwait(false); + } + } + + /// + /// Loads a manifest from JSON format. + /// + /// The file path to load the manifest from. + /// The cancellation token to obey. + /// Optional JSON options to use. + /// The loaded , or null if the manifest could not be loaded. + public async Task LoadAsync(string path, CancellationToken cancel, JsonSerializerOptions? jsonOptions = null) + { + if (!_fileSystem.File.Exists(path)) + { + return null; + } + + jsonOptions = MergeJsonOptions(jsonOptions); + + var file = _fileSystem.File.OpenRead(path); + await using (file.ConfigureAwait(false)) + { + var manifest = await JsonSerializer.DeserializeAsync(file, jsonOptions, cancel) + .ConfigureAwait(false); + + if (manifest is not null) + { + if (manifest.ManifestVersion is not SupportedManifestVersion) + throw new NotSupportedException($"This {nameof(MigrationManifestSerializer)} only supports Manifest version {SupportedManifestVersion}. The manifest being loaded is version {manifest.ManifestVersion}"); + + return manifest.ToMigrationManifest(_localizer, _loggerFactory) as MigrationManifest; + } + + return null; + } + } + } +} diff --git a/src/Tableau.Migration/ExceptionComparer.cs b/src/Tableau.Migration/ExceptionComparer.cs new file mode 100644 index 00000000..9e025d74 --- /dev/null +++ b/src/Tableau.Migration/ExceptionComparer.cs @@ -0,0 +1,74 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Reflection; + +namespace Tableau.Migration +{ + /// + /// Provides methods for comparing exceptions + /// + public class ExceptionComparer : IEqualityComparer + { + /// + public bool Equals(Exception? x, Exception? y) + { + if (x is null && y is null) return true; + if (x is null || y is null) return false; + + // Check if x and y are of the same type + if (x.GetType() != y.GetType()) + { + return false; + } + + // Check if they implement IEquatable for their specific type + Type equatableType = typeof(IEquatable<>).MakeGenericType(x.GetType()); + if (equatableType.IsAssignableFrom(x.GetType())) + { + return (bool?)equatableType.GetMethod("Equals")?.Invoke(x, new object?[] { y }) ?? false; + } + else + { + return x.Message == y.Message; + } + } + + /// + public int GetHashCode(Exception obj) + { + // Check if the object's type overrides GetHashCode + MethodInfo? getHashCodeMethod = obj.GetType().GetMethod("GetHashCode", Type.EmptyTypes); + if (getHashCodeMethod != null && getHashCodeMethod.DeclaringType != typeof(object)) + { + return obj.GetHashCode(); + } + else if (obj is IEquatable) + { + return obj.GetHashCode(); + } + else + { + return obj.Message?.GetHashCode() ?? 0; + } + } + + } +} \ No newline at end of file diff --git a/src/Tableau.Migration/JsonConverters/BuildResponseExceptionJsonConverter.cs b/src/Tableau.Migration/JsonConverters/BuildResponseExceptionJsonConverter.cs new file mode 100644 index 00000000..14736d16 --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/BuildResponseExceptionJsonConverter.cs @@ -0,0 +1,97 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Net; +using System.Text.Json; +using System.Text.Json.Serialization; +using Tableau.Migration.Api.Simulation.Rest.Net.Responses; + +namespace Tableau.Migration.JsonConverters +{ + /// + /// JsonConverter that serializes a . It does not support reading exceptions back in. + /// + internal class BuildResponseExceptionJsonConverter : JsonConverter + { + public override void Write(Utf8JsonWriter writer, BuildResponseException value, JsonSerializerOptions options) + { + writer.WriteStartObject(); + writer.WriteString(nameof(BuildResponseException.StatusCode), value.StatusCode.ToString()); + writer.WriteNumber(nameof(BuildResponseException.SubCode), value.SubCode); + writer.WriteString(nameof(BuildResponseException.Summary), value.Summary); + writer.WriteString(nameof(BuildResponseException.Detail), value.Detail); + JsonWriterUtils.WriteExceptionProperties(ref writer, value); + writer.WriteEndObject(); + } + + + public override BuildResponseException? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + JsonReaderUtils.AssertStartObject(ref reader); + + HttpStatusCode? statusCode = null; + int? subCode = null; + string? summary = null; + string? detail = null; + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.PropertyName) + { + string? propertyName = reader.GetString(); + Guard.AgainstNullOrEmpty(propertyName, nameof(propertyName)); + + reader.Read(); // Move to the property value. + + switch(propertyName) + { + case nameof(BuildResponseException.StatusCode): + var statusCodeStr = reader.GetString(); + Guard.AgainstNull(statusCodeStr, nameof(statusCodeStr)); + statusCode = (HttpStatusCode)Enum.Parse(typeof(HttpStatusCode), statusCodeStr); + break; + + case nameof(BuildResponseException.SubCode): + subCode = reader.GetInt32(); + break; + + case nameof(BuildResponseException.Summary): + summary = reader.GetString(); + Guard.AgainstNull(summary, nameof(summary)); + break; + + case nameof(BuildResponseException.Detail): + detail = reader.GetString(); + Guard.AgainstNull(detail, nameof(detail)); + break; + + default: + break; + } + } + else if (reader.TokenType == JsonTokenType.EndObject) + { + break; // End of the object. + } + } + + return new BuildResponseException(statusCode!.Value, subCode!.Value, summary!, detail!); + } + + } +} diff --git a/src/Tableau.Migration/JsonConverters/Constants.cs b/src/Tableau.Migration/JsonConverters/Constants.cs new file mode 100644 index 00000000..112c920b --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/Constants.cs @@ -0,0 +1,27 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +namespace Tableau.Migration.JsonConverters +{ + internal static class Constants + { + public const string PARTITION = "Partition"; + public const string ENTRIES = "Entries"; + public const string CLASS_NAME = "ClassName"; + public const string EXCEPTION = "Exception"; + } +} diff --git a/src/Tableau.Migration/JsonConverters/ExceptionJsonConverter.cs b/src/Tableau.Migration/JsonConverters/ExceptionJsonConverter.cs new file mode 100644 index 00000000..e86464dd --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/ExceptionJsonConverter.cs @@ -0,0 +1,87 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Tableau.Migration.JsonConverters +{ + /// + /// JsonConverter that serializes and deserializes any type of . + /// + /// The type of the exception to convert. + public class ExceptionJsonConverter : JsonConverter where TException : Exception + { + /// + /// Writes the JSON representation of an Exception object. + /// + /// The to write to. + /// The object to serialize. + /// The to use for serialization. + public override void Write(Utf8JsonWriter writer, TException value, JsonSerializerOptions options) + { + writer.WriteStartObject(); + JsonWriterUtils.WriteExceptionProperties(ref writer, value); + writer.WriteEndObject(); + } + + /// + /// Reads the JSON representation of an Exception object. + /// + /// The to read from. + /// The type of the object to convert. + /// The to use for deserialization. + /// The deserialized object. + public override TException? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + JsonReaderUtils.AssertStartObject(ref reader); + + string? message = null; + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.PropertyName) + { + string? propertyName = reader.GetString(); + Guard.AgainstNullOrEmpty(propertyName, nameof(propertyName)); + + reader.Read(); // Move to the property value. + + if (propertyName == "Message") + { + message = reader.GetString(); + Guard.AgainstNull(message, nameof(message)); // Message could be an empty string, so just check null + } + if (propertyName == "InnerException") + { + reader.Skip(); // Don't read in the inner exceptions + } + } + else if (reader.TokenType == JsonTokenType.EndObject) + { + break; // End of the object. + } + } + + // Message must be deserialized by now + Guard.AgainstNull(message, nameof(message)); + + return (TException)Activator.CreateInstance(typeof(TException), message)!; + } + } +} diff --git a/src/Tableau.Migration/JsonConverters/ExceptionJsonConverterFactory.cs b/src/Tableau.Migration/JsonConverters/ExceptionJsonConverterFactory.cs new file mode 100644 index 00000000..7282a0ce --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/ExceptionJsonConverterFactory.cs @@ -0,0 +1,42 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Tableau.Migration.JsonConverters +{ + /// + /// JsonConverterFactory that creates converters for any type of . + /// + public class ExceptionJsonConverterFactory : JsonConverterFactory + { + /// + public override bool CanConvert(Type typeToConvert) + { + return typeof(Exception).IsAssignableFrom(typeToConvert); + } + + /// + public override JsonConverter CreateConverter(Type typeToConvert, JsonSerializerOptions options) + { + var converterType = typeof(ExceptionJsonConverter<>).MakeGenericType(typeToConvert); + return (JsonConverter)Activator.CreateInstance(converterType)!; + } + } +} diff --git a/src/Tableau.Migration/JsonConverters/Exceptions/MismatchException.cs b/src/Tableau.Migration/JsonConverters/Exceptions/MismatchException.cs new file mode 100644 index 00000000..1b1fd685 --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/Exceptions/MismatchException.cs @@ -0,0 +1,53 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; + +namespace Tableau.Migration.JsonConverters.Exceptions +{ + /// + /// The exception that is thrown when a that was serialized and then deserialized but did not match the initial Manifest. + /// + /// This means that the either the serializer or deserializer has a bug. + public class MismatchException : Exception + { + /// + /// The exception that is thrown when a that was serialized and then deserialized but did not match the initial Manifest. + /// + /// This means that the either the serializer or deserializer has a bug. + public MismatchException() + { + } + + /// + /// The exception that is thrown when a that was serialized and then deserialized but did not match the initial Manifest. + /// + /// This means that the either the serializer or deserializer has a bug. + public MismatchException(string message) : base(message) + { + } + + /// + /// The exception that is thrown when a that was serialized and then deserialized but did not match the initial Manifest. + /// + /// This means that the either the serializer or deserializer has a bug. + public MismatchException(string message, Exception innerException) + : base(message, innerException) + { + } + } +} \ No newline at end of file diff --git a/src/Tableau.Migration/JsonConverters/FailedJobExceptionJsonConverter.cs b/src/Tableau.Migration/JsonConverters/FailedJobExceptionJsonConverter.cs new file mode 100644 index 00000000..b3ba354a --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/FailedJobExceptionJsonConverter.cs @@ -0,0 +1,73 @@ +using System; +using System.Text.Json; +using System.Text.Json.Serialization; +using Tableau.Migration.Api.Models; + +namespace Tableau.Migration.JsonConverters +{ + /// + /// Provides a custom JSON converter for objects, allowing for custom serialization and deserialization logic. + /// + public class FailedJobExceptionJsonConverter : JsonConverter + { + /// + /// Reads and converts the JSON to type . + /// + /// The reader to deserialize objects or value types. + /// The type of object to convert. + /// Options to control the behavior during reading. + /// A object deserialized from JSON. + public override FailedJobException Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + IJob? failedJob = null; + string? exceptionMessage = null; + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.PropertyName) + { + var propertyName = reader.GetString(); + reader.Read(); // Move to the property value. + + switch (propertyName) + { + case nameof(FailedJobException.FailedJob): + failedJob = JsonSerializer.Deserialize(ref reader, options); + break; + + case nameof(FailedJobException.Message): + exceptionMessage = reader.GetString(); + break; + } + } + else if (reader.TokenType == JsonTokenType.EndObject) + { + break; // End of the object. + } + } + + Guard.AgainstNull(exceptionMessage, nameof(exceptionMessage)); + Guard.AgainstNull(failedJob, nameof(failedJob)); + + return new FailedJobException(failedJob, exceptionMessage); + } + + /// + /// Writes a specified object to JSON. + /// + /// The writer to serialize objects or value types. + /// The value to serialize. + /// Options to control the behavior during writing. + public override void Write(Utf8JsonWriter writer, FailedJobException value, JsonSerializerOptions options) + { + writer.WriteStartObject(); + + writer.WritePropertyName(nameof(FailedJobException.FailedJob)); + JsonSerializer.Serialize(writer, value.FailedJob, options); + + writer.WriteString(nameof(FailedJobException.Message), value.Message); + + writer.WriteEndObject(); + } + } +} diff --git a/src/Tableau.Migration/JsonConverters/JobJsonConverter.cs b/src/Tableau.Migration/JsonConverters/JobJsonConverter.cs new file mode 100644 index 00000000..a4da43c8 --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/JobJsonConverter.cs @@ -0,0 +1,175 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; +using Tableau.Migration.Api.Models; +using Tableau.Migration.Api.Rest.Models.Responses; + +namespace Tableau.Migration.JsonConverters +{ + /// + /// JsonConverter that serializes a . + /// + internal class JobJsonConverter : JsonConverter + { + public JobJsonConverter() + { } + + public override void Write(Utf8JsonWriter writer, IJob value, JsonSerializerOptions options) + { + writer.WriteStartObject(); + + writer.WriteString(nameof(IJob.Id), value.Id); + writer.WriteString(nameof(IJob.Type), value.Type); + writer.WriteString(nameof(IJob.CreatedAtUtc), value.CreatedAtUtc); + + if (value.UpdatedAtUtc.HasValue) + { + writer.WriteString(nameof(IJob.UpdatedAtUtc), value.UpdatedAtUtc.Value); + } + + if (value.CompletedAtUtc.HasValue) + { + writer.WriteString(nameof(IJob.CompletedAtUtc), value.CompletedAtUtc.Value); + } + + writer.WriteNumber(nameof(IJob.ProgressPercentage), value.ProgressPercentage); + writer.WriteNumber(nameof(IJob.FinishCode), value.FinishCode); + + // Serializing StatusNotes + writer.WriteStartArray(nameof(IJob.StatusNotes)); + foreach (var statusNote in value.StatusNotes) + { + writer.WriteStartObject(); + writer.WriteString(nameof(IStatusNote.Type), statusNote.Type); + writer.WriteString(nameof(IStatusNote.Value), statusNote.Value); + writer.WriteString(nameof(IStatusNote.Text), statusNote.Text); + writer.WriteEndObject(); + } + writer.WriteEndArray(); + + writer.WriteEndObject(); + } + + + + public override IJob? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType != JsonTokenType.StartObject) + { + throw new JsonException("Expected StartObject token"); + } + + var jobResponse = new JobResponse.JobType(); + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + { + break; // End of the job object. + } + + if (reader.TokenType == JsonTokenType.PropertyName) + { + var propertyName = reader.GetString(); + reader.Read(); // Move to the property value. + + switch (propertyName) + { + case nameof(IJob.Id): + jobResponse.Id = reader.GetGuid(); + break; + case nameof(IJob.Type): + jobResponse.Type = reader.GetString(); + break; + case nameof(IJob.CreatedAtUtc): + jobResponse.CreatedAt = reader.GetDateTime().ToString("o"); + break; + case nameof(IJob.UpdatedAtUtc): + jobResponse.UpdatedAt = reader.GetDateTime().ToString("o"); + break; + case nameof(IJob.CompletedAtUtc): + jobResponse.CompletedAt = reader.GetDateTime().ToString("o"); + break; + case nameof(IJob.ProgressPercentage): + jobResponse.Progress = reader.GetInt32(); + break; + case nameof(IJob.FinishCode): + jobResponse.FinishCode = reader.GetInt32(); + break; + case nameof(IJob.StatusNotes): + jobResponse.StatusNotes = ReadStatusNotes(ref reader); + break; + default: + reader.Skip(); // Skip unknown properties. + break; + } + } + } + + var jobResponseWrapper = new JobResponse { Item = jobResponse }; + return new Job(jobResponseWrapper); + } + + private JobResponse.JobType.StatusNoteType[] ReadStatusNotes(ref Utf8JsonReader reader) + { + if (reader.TokenType != JsonTokenType.StartArray) + { + throw new JsonException("Expected StartArray token for StatusNotes"); + } + + var statusNotes = new List(); + + while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) + { + if (reader.TokenType == JsonTokenType.StartObject) + { + var statusNote = new JobResponse.JobType.StatusNoteType(); + while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) + { + if (reader.TokenType == JsonTokenType.PropertyName) + { + var propertyName = reader.GetString(); + reader.Read(); // Move to the property value. + + switch (propertyName) + { + case nameof(IStatusNote.Type): + statusNote.Type = reader.GetString(); + break; + case nameof(IStatusNote.Value): + statusNote.Value = reader.GetString(); + break; + case nameof(IStatusNote.Text): + statusNote.Text = reader.GetString(); + break; + default: + reader.Skip(); // Skip unknown properties. + break; + } + } + } + statusNotes.Add(statusNote); + } + } + + return statusNotes.ToArray(); + } + } +} diff --git a/src/Tableau.Migration/JsonConverters/JsonReaderUtils.cs b/src/Tableau.Migration/JsonConverters/JsonReaderUtils.cs new file mode 100644 index 00000000..3430d349 --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/JsonReaderUtils.cs @@ -0,0 +1,104 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; + +namespace Tableau.Migration.JsonConverters +{ + internal static class JsonReaderUtils + { + /// + /// Verify that last json token was a + /// + internal static void AssertPropertyName(ref Utf8JsonReader reader, string? expected = null) + { + if (reader.TokenType != JsonTokenType.PropertyName) throw new JsonException("Expected property name"); + if (expected != null) + { + var actual = reader.GetString(); + if (!string.Equals(actual, expected, StringComparison.Ordinal)) + { + throw new JsonException($"Property value did not match expectation. Expected '{expected}' but got '{actual}'"); + } + } + } + + /// + /// Verify that last json token was a + /// + internal static void AssertStartArray(ref Utf8JsonReader reader) + { + if (reader.TokenType != JsonTokenType.StartArray) throw new JsonException("Expected start of array"); + } + + /// + /// Verify that last json token was a + /// + internal static void AssertEndArray(ref Utf8JsonReader reader) + { + if (reader.TokenType != JsonTokenType.EndArray) throw new JsonException("Expected end of array"); + } + + /// + /// Verify that last json token was a + /// + internal static void AssertStartObject(ref Utf8JsonReader reader) + { + if (reader.TokenType != JsonTokenType.StartObject) throw new JsonException("Expected start of object"); + } + + /// + /// Verify that last json token was a + /// + internal static void AssertEndObject(ref Utf8JsonReader reader) + { + if (reader.TokenType != JsonTokenType.EndObject) throw new JsonException("Expected end of object"); + } + + /// + /// Read the next json token and verify that it's a + /// + internal static void ReadAndAssertPropertyName(ref Utf8JsonReader reader, string? expected = null) + { + reader.Read(); + AssertPropertyName(ref reader, expected); + } + + /// + /// Read the next json token and verify that it's a + /// + internal static void ReadAndAssertStartArray(ref Utf8JsonReader reader) + { + reader.Read(); + AssertStartArray(ref reader); + } + + /// + /// Read the next json token and verify that it's a + /// + internal static void ReadAndAssertStartObject(ref Utf8JsonReader reader) + { + reader.Read(); + AssertStartObject(ref reader); + } + } +} diff --git a/src/Tableau.Migration/JsonConverters/JsonWriterUtils.cs b/src/Tableau.Migration/JsonConverters/JsonWriterUtils.cs new file mode 100644 index 00000000..22df0541 --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/JsonWriterUtils.cs @@ -0,0 +1,48 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Text.Json; + +namespace Tableau.Migration.JsonConverters +{ + internal static class JsonWriterUtils + { + internal static void WriteExceptionProperties(ref Utf8JsonWriter writer, Exception value) + { + writer.WriteString("Message", value.Message); + writer.WriteString("Type", value.GetType().FullName); + writer.WriteString("Source", value.Source); + writer.WriteString("StackTrace", value.StackTrace); + writer.WriteString("HelpLink", value.HelpLink); // Include HelpLink + writer.WriteNumber("HResult", value.HResult); // Include HResult + + // Leaving this at 1 level of depth. + if (value.InnerException != null) + { + writer.WriteStartObject("InnerException"); + writer.WriteString("Message", value.InnerException.Message); + writer.WriteString("Type", value.InnerException.GetType().FullName); + writer.WriteString("Source", value.InnerException.Source); + writer.WriteString("StackTrace", value.InnerException.StackTrace); + writer.WriteString("HelpLink", value.InnerException.HelpLink); // Include HelpLink for InnerException + writer.WriteNumber("HResult", value.InnerException.HResult); // Include HResult for InnerException + writer.WriteEndObject(); + } + } + } +} diff --git a/src/Tableau.Migration/JsonConverters/PythonExceptionConverter.cs b/src/Tableau.Migration/JsonConverters/PythonExceptionConverter.cs new file mode 100644 index 00000000..7b78f77a --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/PythonExceptionConverter.cs @@ -0,0 +1,106 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; +using Python.Runtime; + +namespace Tableau.Migration.JsonConverters +{ + /// + /// JsonConverter that serializes a . + /// + public class PythonExceptionConverter : JsonConverter + { + // **NOTE** + // I am not convinced this works. + // I've tried to create a PythonException manually to pass into the write and read method, but I can't create one. + // Every time I try I get an exception through in the Python.NET layer I get an exception thrown in the Python.NET layer. + // + // The other option is to actually run python code that will produce an PythonException, except that requires + // scaffolding as it requires the python library to be imported to PythonRuntime. + // See: https://github.com/pythonnet/pythonnet#embedding-python-in-net + // You must set Runtime.PythonDLL property or PYTHONNET_PYDLL environment variable starting with version 3.0, + // otherwise you will receive BadPythonDllException (internal, derived from MissingMethodException) upon calling Initialize. + // Typical values are python38.dll (Windows), libpython3.8.dylib (Mac), libpython3.8.so (most other Unix-like operating systems). + // This is difficult because I have no way to get a libpython. on mac. + // I could do this for just windows, but that would require us to add a python38.dll to the repo. + // + // I have tested that the "write" works manually. This is because I had a bug in the Python.TestApplication in a hook, that produced it. + // I have no idea how it actually made it into the manifest though, and at this point, I'm out of time. + // I have opened a user story to work on this some more at a later time. + + /// + /// Reads the JSON representation of the object. + /// + /// The reader to read from. + /// The type of the object. + /// The serializer options. + /// The deserialized object. + public override PythonException? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + JsonReaderUtils.AssertStartObject(ref reader); + + string? message = null; + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.PropertyName) + { + string? propertyName = reader.GetString(); + Guard.AgainstNullOrEmpty(propertyName, nameof(propertyName)); + + reader.Read(); // Move to the property value. + + if (propertyName == "Message") + { + message = reader.GetString(); + Guard.AgainstNull(message, nameof(message)); // Message could be an empty string, so just check null + } + } + else if (reader.TokenType == JsonTokenType.EndObject) + { + break; // End of the object. + } + } + + // Message must be deserialized by now + Guard.AgainstNull(message, nameof(message)); + var ret = new PythonException(PyType.Get(typeof(string)), PyType.None, null, message, null); + return ret; + } + + /// + /// Writes the JSON representation of the object. + /// + /// The writer to write to. + /// The object to write. + /// The serializer options. + public override void Write(Utf8JsonWriter writer, PythonException value, JsonSerializerOptions options) + { + if (value is PythonException pyException) + { + writer.WriteStartObject(); + writer.WriteString("ClassName", typeof(PythonException).FullName); + writer.WriteString("Message", pyException.Format()); + writer.WriteEndObject(); + return; + } + } + } +} diff --git a/src/Tableau.Migration/JsonConverters/RestExceptionJsonConverter.cs b/src/Tableau.Migration/JsonConverters/RestExceptionJsonConverter.cs new file mode 100644 index 00000000..e22d638b --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/RestExceptionJsonConverter.cs @@ -0,0 +1,145 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Text.Json; +using System.Text.Json.Serialization; +using Tableau.Migration.Api.Rest; +using Tableau.Migration.Api.Rest.Models; +using Tableau.Migration.JsonConverters.SerializableObjects; + +namespace Tableau.Migration.JsonConverters +{ + /// + /// Represents a collection of serializable entries, organized by a string key and a list of as the value. + /// This class extends to facilitate serialization and deserialization of migration manifest entries. + /// + public class RestExceptionJsonConverter : JsonConverter + { + /// + /// Reads and converts the JSON to type . + /// + /// The reader to deserialize objects or value types. + /// The type of object to convert. + /// Options to control the behavior during reading. + /// A object deserialized from JSON. + public override RestException Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + HttpMethod? httpMethod = null; + Uri? requestUri = null; + string? code = null; + string? detail = null; + string? summary = null; + string? exceptionMessage = null; + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.PropertyName) + { + var propertyName = reader.GetString(); + reader.Read(); // Move to the property value. + + switch (propertyName) + { + case nameof(RestException.HttpMethod): + var method = reader.GetString(); + if (method != null) + { + httpMethod = new HttpMethod(method); + } + break; + + case nameof(RestException.RequestUri): + var uriString = reader.GetString(); + if (uriString != null) + { + requestUri = new Uri(uriString); + } + break; + + case nameof(RestException.Code): + code = reader.GetString(); + break; + + case nameof(RestException.Detail): + detail = reader.GetString(); + break; + + case nameof(RestException.Summary): + summary = reader.GetString(); + break; + + case nameof(RestException.Message): + exceptionMessage = reader.GetString(); + break; + } + } + else if (reader.TokenType == JsonTokenType.EndObject) + { + break; // End of the object. + } + } + + Guard.AgainstNull(exceptionMessage, nameof(exceptionMessage)); + + // Use the internal constructor for deserialization + return new RestException(httpMethod, requestUri, new Error { Code = code, Detail = detail, Summary = summary }, exceptionMessage); + } + + /// + /// Writes a specified object to JSON. + /// + /// The writer to serialize objects or value types. + /// The value to serialize. + /// Options to control the behavior during writing. + public override void Write(Utf8JsonWriter writer, RestException value, JsonSerializerOptions options) + { + writer.WriteStartObject(); + + if (value.HttpMethod != null) + { + writer.WriteString(nameof(RestException.HttpMethod), value.HttpMethod.Method); + } + + if (value.RequestUri != null) + { + writer.WriteString(nameof(RestException.RequestUri), value.RequestUri.ToString()); + } + + if (value.Code != null) + { + writer.WriteString(nameof(RestException.Code), value.Code); + } + + if (value.Detail != null) + { + writer.WriteString(nameof(RestException.Detail), value.Detail); + } + + if (value.Summary != null) + { + writer.WriteString(nameof(RestException.Summary), value.Summary); + } + + JsonWriterUtils.WriteExceptionProperties(ref writer, value); + + writer.WriteEndObject(); + } + } +} diff --git a/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableContentLocation.cs b/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableContentLocation.cs new file mode 100644 index 00000000..b9fe5e5b --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableContentLocation.cs @@ -0,0 +1,106 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Linq; +using Tableau.Migration.JsonConverters.Exceptions; + +namespace Tableau.Migration.JsonConverters.SerializableObjects +{ + /// + /// Represents a JSON serializable content location, providing details about the location of content within a migration context. + /// + public class SerializableContentLocation + { + /// + /// Gets or sets the path segments that make up the content location. + /// + public string[]? PathSegments { get; set; } + + /// + /// Gets or sets the path separator used between path segments. + /// + public string? PathSeparator { get; set; } + + /// + /// Gets or sets the full path constructed from the path segments and separator. + /// + public string? Path { get; set; } + + /// + /// Gets or sets the name of the content at this location. + /// + public string? Name { get; set; } + + /// + /// Gets or sets a value indicating whether the content location is empty. + /// + public bool IsEmpty { get; set; } + + /// + /// Initializes a new instance of the class. + /// + public SerializableContentLocation() { } + + /// + /// Initializes a new instance of the class with details from an existing . + /// + /// The content location to serialize. + internal SerializableContentLocation(ContentLocation location) + { + PathSegments = location.PathSegments.ToArray(); + PathSeparator = location.PathSeparator; + Path = location.Path; + Name = location.Name; + IsEmpty = location.IsEmpty; + } + + /// + /// Throw exception if any values are still null + /// + public void VerifyDeseralization() + { + Guard.AgainstNull(PathSegments, nameof(PathSegments)); + Guard.AgainstNull(PathSeparator, nameof(PathSeparator)); + Guard.AgainstNull(Path, nameof(Path)); + Guard.AgainstNull(Name, nameof(Name)); + + var expectedName = PathSegments.LastOrDefault() ?? string.Empty; + if (!string.Equals(Name, expectedName, StringComparison.Ordinal)) + throw new MismatchException($"{nameof(Name)} should be {expectedName} but is {Name}."); + + var expectedPath = string.Join(PathSeparator, PathSegments); + if (!string.Equals(Path, expectedPath, StringComparison.Ordinal)) + throw new MismatchException($"{nameof(Path)} should be {expectedPath} but is {Path}."); + + var expectedIsEmpty = (PathSegments.Length == 0); + if (IsEmpty != expectedIsEmpty) + throw new MismatchException($"{nameof(IsEmpty)} should be {expectedIsEmpty} but is {IsEmpty}."); + } + + /// + /// Returns the current object as a + /// + /// + public ContentLocation AsContentLocation() + { + VerifyDeseralization(); + var ret = new ContentLocation(PathSeparator!, PathSegments!); + return ret; + } + } +} diff --git a/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableContentReference.cs b/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableContentReference.cs new file mode 100644 index 00000000..d2461cba --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableContentReference.cs @@ -0,0 +1,93 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using Tableau.Migration.Content; + +namespace Tableau.Migration.JsonConverters.SerializableObjects +{ + /// + /// Represents a JSON serializable content reference. + /// + public class SerializableContentReference + { + /// + /// Gets or sets the unique identifier for the content. + /// + public string? Id { get; set; } + + /// + /// Gets or sets the URL associated with the content. + /// + public string? ContentUrl { get; set; } + + /// + /// Gets or sets the location information for the content. + /// + public SerializableContentLocation? Location { get; set; } + + /// + /// Gets or sets the name of the content. + /// + public string? Name { get; set; } + + /// + /// Initializes a new instance of the class. + /// + public SerializableContentReference() { } + + /// + /// Initializes a new instance of the class with details from an . + /// + /// The content reference to serialize. + internal SerializableContentReference(IContentReference content) + { + Id = content.Id.ToString(); + ContentUrl = content.ContentUrl; + Location = new SerializableContentLocation(content.Location); + Name = content.Name; + } + + /// + /// Throw exception if any values are still null + /// + public void VerifyDeserialization() + { + Guard.AgainstNull(Id, nameof(Id)); + Guard.AgainstNull(ContentUrl, nameof(ContentUrl)); + Guard.AgainstNull(Location, nameof(Location)); + Guard.AgainstNull(Name, nameof(Name)); + } + + /// + /// Returns the current item as a + /// + /// + public ContentReferenceStub AsContentReferenceStub() + { + VerifyDeserialization(); + + var ret = new ContentReferenceStub( + Guid.Parse(Id!), + ContentUrl!, + Location!.AsContentLocation(), + Name!); + + return ret; + } + } +} \ No newline at end of file diff --git a/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableEntryCollection.cs b/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableEntryCollection.cs new file mode 100644 index 00000000..32fcec62 --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableEntryCollection.cs @@ -0,0 +1,50 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Tableau.Migration.Engine.Manifest; + +namespace Tableau.Migration.JsonConverters.SerializableObjects +{ + /// + /// Represents a collection of serializable entries, organized by a string key and a list of as the value. + /// This class extends to facilitate serialization and deserialization of migration manifest entries. + /// + public class SerializableEntryCollection : Dictionary> + { + /// + /// Initializes a new instance of the class. + /// + public SerializableEntryCollection() { } + + /// + /// Initializes a new instance of the class with an existing dictionary of entries. + /// + /// The dictionary containing the initial entries for the collection. + public SerializableEntryCollection(Dictionary> entries) + { + foreach (var entry in entries) + { + Add(entry.Key, entry.Value); + } + } + } +} diff --git a/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableException.cs b/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableException.cs new file mode 100644 index 00000000..72155a8d --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableException.cs @@ -0,0 +1,47 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; + +namespace Tableau.Migration.JsonConverters.SerializableObjects +{ + /// + /// Represents a serializable version of an exception, allowing exceptions to be serialized into JSON format. + /// + public class SerializableException + { + /// + /// Gets or sets the class name of the exception. + /// + public string? ClassName { get; set; } + + /// + /// Gets or sets the exception object. This property is not serialized and is used only for internal purposes. + /// + public Exception? Error { get; set; } = null; + + /// + /// Initializes a new instance of the class using the specified exception. + /// + /// The exception to serialize. + internal SerializableException(Exception ex) + { + ClassName = ex.GetType().FullName; + Error = ex; + } + } +} diff --git a/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableManifestEntry.cs b/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableManifestEntry.cs new file mode 100644 index 00000000..bfa66b46 --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableManifestEntry.cs @@ -0,0 +1,147 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using Tableau.Migration.Engine.Manifest; + +namespace Tableau.Migration.JsonConverters.SerializableObjects +{ + /// + /// Represents a JSON serializable entry in a migration manifest. This class implements + /// to allow for easy conversion between the manifest entry and its JSON representation. + /// + public class SerializableManifestEntry : IMigrationManifestEntry + { + /// + /// Gets or sets the source content reference. + /// + public SerializableContentReference? Source { get; set; } + + /// + /// Gets or sets the mapped location for the content. + /// + public SerializableContentLocation? MappedLocation { get; set; } + + /// + /// Gets or sets the destination content reference. + /// + public SerializableContentReference? Destination { get; set; } + + /// + /// Gets or sets the status of the migration for this entry. + /// + public int Status { get; set; } + + /// + /// Gets or sets a value indicating whether the content has been migrated. + /// + public bool HasMigrated { get; set; } + + /// + /// Gets or sets the list of errors encountered during the migration of this entry. + /// + public List? Errors { get; set; } + + /// + /// Initializes a new instance of the class. + /// + public SerializableManifestEntry() { } + + /// + /// Initializes a new instance of the class with details from an . + /// + /// The migration manifest entry to serialize. + internal SerializableManifestEntry(IMigrationManifestEntry entry) + { + Source = new SerializableContentReference(entry.Source); + MappedLocation = new SerializableContentLocation(entry.MappedLocation); + Destination = entry.Destination == null ? null : new SerializableContentReference(entry.Destination); + Status = (int)entry.Status; + HasMigrated = entry.HasMigrated; + + Errors = entry.Errors.Select(e => new SerializableException(e)).ToList(); + } + + IContentReference IMigrationManifestEntry.Source => Source!.AsContentReferenceStub(); + + ContentLocation IMigrationManifestEntry.MappedLocation => MappedLocation!.AsContentLocation(); + + IContentReference? IMigrationManifestEntry.Destination => Destination?.AsContentReferenceStub(); + + MigrationManifestEntryStatus IMigrationManifestEntry.Status => (MigrationManifestEntryStatus)Status; + + bool IMigrationManifestEntry.HasMigrated => HasMigrated; + + IReadOnlyList IMigrationManifestEntry.Errors + { + get + { + if (Errors == null) + { + return Array.Empty(); + } + else + { + return Errors.Select(e => e.Error).ToImmutableArray(); + } + } + } + + /// + /// Sets the list of errors encountered during the migration of this entry. + /// + /// The list of errors to set. + public void SetErrors(List errors) + { + Errors = errors.Select(e => new SerializableException(e)).ToList(); + } + + /// + /// Throw exception if any values are still null + /// + public void VerifyDeseralization() + { + // Destination can be null, so we shouldn't do a nullability check on it + + Guard.AgainstNull(Source, nameof(Source)); + Guard.AgainstNull(MappedLocation, nameof(MappedLocation)); + + Source.VerifyDeserialization(); + MappedLocation.VerifyDeseralization(); + } + + /// + /// Returns current object as a + /// + /// + public IMigrationManifestEntry AsMigrationManifestEntry(IMigrationManifestEntryBuilder partition) + { + VerifyDeseralization(); + var ret = new MigrationManifestEntry(partition, this); + + return ret; + } + + /// + public bool Equals(IMigrationManifestEntry? other) + => MigrationManifestEntry.Equals(this, other); + } +} \ No newline at end of file diff --git a/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableMigrationManifest.cs b/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableMigrationManifest.cs new file mode 100644 index 00000000..aa6cef49 --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableMigrationManifest.cs @@ -0,0 +1,129 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using Microsoft.Extensions.Logging; +using Tableau.Migration.Engine.Manifest; +using Tableau.Migration.Resources; + +namespace Tableau.Migration.JsonConverters.SerializableObjects +{ + /// + /// Represents a serializable version of a migration manifest, which can be used for JSON serialization and deserialization. + /// + public class SerializableMigrationManifest + { + /// + /// Gets or sets the unique identifier for the migration plan. + /// + public Guid? PlanId { get; set; } + + /// + /// Gets or sets the unique identifier for the migration. + /// + public Guid? MigrationId { get; set; } + + /// + /// Gets or sets the list of errors encountered during the migration process. + /// + public List? Errors { get; set; } = new List(); + + /// + /// Gets or sets the collection of entries that are part of the migration manifest. + /// + public SerializableEntryCollection? Entries { get; set; } = new(); + + /// + /// Gets or sets the version of the migration manifest. + /// + public uint? ManifestVersion { get; set; } + + /// + /// Initializes a new instance of the class. + /// + public SerializableMigrationManifest() { } + + /// + /// Initializes a new instance of the class with details from an . + /// + /// The migration manifest to serialize. + public SerializableMigrationManifest(IMigrationManifest manifest) + { + PlanId = manifest.PlanId; + MigrationId = manifest.MigrationId; + ManifestVersion = manifest.ManifestVersion; + + Errors = manifest.Errors.Select(e => new SerializableException(e)).ToList(); + + foreach (var partitionType in manifest.Entries.GetPartitionTypes()) + { + Guard.AgainstNull(partitionType, nameof(partitionType)); + Guard.AgainstNullOrEmpty(partitionType.FullName, nameof(partitionType.FullName)); + + Entries.Add(partitionType!.FullName, manifest.Entries.ForContentType(partitionType).Select(entry => new SerializableManifestEntry(entry)).ToList()); + } + } + + /// + /// Converts the serializable migration manifest back into an instance. + /// + /// The shared resources localizer. + /// The logger factory. + /// An instance of . + public IMigrationManifest ToMigrationManifest(ISharedResourcesLocalizer localizer, ILoggerFactory loggerFactory) + { + VerifyDeserialization(); + + // Create the manifest to return + var manifest = new MigrationManifest(localizer, loggerFactory, PlanId!.Value, MigrationId!.Value); + + // Get the Tableau.Migration assembly to get the type from later + var loadedAssemblies = AppDomain.CurrentDomain.GetAssemblies(); + var tableauMigrationAssembly = loadedAssemblies.Where(a => a.ManifestModule.Name == "Tableau.Migration.dll").First(); + + // Copy the entries to the manifest + foreach (var partitionTypeStr in Entries!.Keys) + { + var partitionType = tableauMigrationAssembly.GetType(partitionTypeStr); + Guard.AgainstNull(partitionType, nameof(partitionType)); + + var partition = manifest.Entries.GetOrCreatePartition(partitionType); + + var partitionEntries = Entries.GetValueOrDefault(partitionTypeStr); + Guard.AgainstNull(partitionEntries, nameof(partitionEntries)); + + partition.CreateEntries(partitionEntries.ToImmutableArray()); + } + + manifest.AddErrors(Errors!.Where(e => e.Error is not null).Select(e => e.Error)!); + + return manifest; + } + + internal void VerifyDeserialization() + { + Guard.AgainstNull(PlanId, nameof(PlanId)); + Guard.AgainstNull(MigrationId, nameof(MigrationId)); + Guard.AgainstNull(Errors, nameof(Errors)); + Guard.AgainstNull(Entries, nameof(Entries)); + Guard.AgainstNull(ManifestVersion, nameof(ManifestVersion)); + } + } +} diff --git a/src/Tableau.Migration/JsonConverters/SerializedExceptionJsonConverter.cs b/src/Tableau.Migration/JsonConverters/SerializedExceptionJsonConverter.cs new file mode 100644 index 00000000..210add84 --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/SerializedExceptionJsonConverter.cs @@ -0,0 +1,152 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; +using Tableau.Migration.Api.Models; +using Tableau.Migration.JsonConverters.SerializableObjects; + + +namespace Tableau.Migration.JsonConverters +{ + /// + /// JsonConverter that de/serializes a . + /// + public class SerializedExceptionJsonConverter : JsonConverter + { + internal static string GetNamespace(string fullTypeName) + { + if (string.IsNullOrEmpty(fullTypeName)) + { + throw new ArgumentException("The type name cannot be null or empty.", nameof(fullTypeName)); + } + + int lastDotIndex = fullTypeName.LastIndexOf('.'); + if (lastDotIndex == -1) + { + throw new ArgumentException("The type name does not contain a namespace.", nameof(fullTypeName)); + } + + return fullTypeName.Substring(0, lastDotIndex); + } + + /// + /// Reads a from JSON. + /// + /// The to read from. + /// The type of the object to convert. + /// The to use for deserialization. + /// The deserialized . + public override SerializableException? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + SerializableException? ret = null; + + // Check if the reader has never been read from + if (reader.TokenStartIndex == 0 && reader.CurrentDepth == 0) + { + reader.Read(); + } + + JsonReaderUtils.AssertStartObject(ref reader); + + while (reader.Read()) + { + // Make sure it starts with "ClassName" + JsonReaderUtils.AssertPropertyName(ref reader, Constants.CLASS_NAME); + + // Read the type of exception class this is + reader.Read(); + + string exceptionTypeStr = reader.GetString() ?? ""; + + string exceptionNamespace = GetNamespace(exceptionTypeStr); + + Type? exceptionType = null; + if (exceptionNamespace == "System") + { + exceptionType = Type.GetType($"{exceptionTypeStr}"); + } + else + { + exceptionType = Type.GetType($"{exceptionTypeStr}, {exceptionNamespace}"); + } + + // Check if this is a built-in exception type + if (exceptionType is null) + { + // exception type is not a built in type, looking through Tableau.Migration.dll + exceptionType = typeof(FailedJobException).Assembly.GetType(exceptionTypeStr); + if (exceptionType is null) + { + if (exceptionTypeStr == "Python.Runtime.PythonException") + { + exceptionType = typeof(Python.Runtime.PythonException); + } + else if (exceptionType != typeof(Python.Runtime.PythonException)) + { + throw new InvalidOperationException($"Unable to get type of '{exceptionTypeStr}'"); + } + + } + } + + // Make sure the next property is the Exception + JsonReaderUtils.ReadAndAssertPropertyName(ref reader, Constants.EXCEPTION); + + // Deserialize the exception + reader.Read(); + var ex = JsonSerializer.Deserialize(ref reader, exceptionType, options) as Exception; + + Guard.AgainstNull(ex, nameof(ex)); + + ret = new SerializableException(ex); + + JsonReaderUtils.AssertEndObject(ref reader); + reader.Read(); + break; + } + + return ret; + } + + /// + /// Writes a to JSON. + /// + /// The to write to. + /// The to write. + /// The to use for serialization. + public override void Write(Utf8JsonWriter writer, SerializableException value, JsonSerializerOptions options) + { + Guard.AgainstNullOrEmpty(value.ClassName, nameof(value.ClassName)); + Guard.AgainstNull(value.Error, nameof(value.Error)); + + // Start our serialized Exception object + writer.WriteStartObject(); + + // Save the type of exception it is + writer.WriteString(Constants.CLASS_NAME, value.ClassName); + + // Save the exception itself + writer.WritePropertyName(Constants.EXCEPTION); + JsonSerializer.Serialize(writer, value.Error, value.Error.GetType(), options); + + // End of serialized exception object + writer.WriteEndObject(); + } + } +} diff --git a/src/Tableau.Migration/JsonConverters/TimeoutJobExceptionJsonConverter.cs b/src/Tableau.Migration/JsonConverters/TimeoutJobExceptionJsonConverter.cs new file mode 100644 index 00000000..6c42490b --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/TimeoutJobExceptionJsonConverter.cs @@ -0,0 +1,83 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; +using Tableau.Migration.Api.Models; + +namespace Tableau.Migration.JsonConverters +{ + /// + /// JsonConverter that serializes a . It does not support reading exceptions back in. + /// + internal class TimeoutJobExceptionJsonConverter : JsonConverter + { + public TimeoutJobExceptionJsonConverter() + { } + + public override void Write(Utf8JsonWriter writer, TimeoutJobException value, JsonSerializerOptions options) + { + writer.WriteStartObject(); + + JsonWriterUtils.WriteExceptionProperties(ref writer, value); + + // Serialize the Job property if it's not null + if (value.Job != null) + { + writer.WritePropertyName("Job"); + JsonSerializer.Serialize(writer, value.Job, options); + } + + writer.WriteEndObject(); + } + + public override TimeoutJobException? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + IJob? job = null; + string? message = null; + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.PropertyName) + { + var propertyName = reader.GetString(); + reader.Read(); // Move to the property value. + + if (propertyName == "Job") + { + job = JsonSerializer.Deserialize(ref reader, options); + } + else if(propertyName == "Message") + { + message = reader.GetString(); + } + } + else if (reader.TokenType == JsonTokenType.EndObject) + { + break; // End of the object. + } + } + + Guard.AgainstNull(message, nameof(message)); // Message could be an empty string, so just check null + + return new TimeoutJobException(job, message); + } + + + } +} diff --git a/tests/Python.ExampleApplication.Tests/Python.ExampleApplication.Tests.pyproj b/tests/Python.ExampleApplication.Tests/Python.ExampleApplication.Tests.pyproj new file mode 100644 index 00000000..61382e09 --- /dev/null +++ b/tests/Python.ExampleApplication.Tests/Python.ExampleApplication.Tests.pyproj @@ -0,0 +1,60 @@ + + + Debug + 2.0 + {b1884017-8e25-4a26-8c89-d9d880cfa392} + + + + + ..\..\src\Python\dist + . + . + Python.ExampleApplication.Tests + Python.ExampleApplication.Tests + MSBuild|env|$(MSBuildProjectFullPath) + False + Pytest + + + true + false + + + true + false + + + + + + + + + + + + + + + + + env + 3.11 + env (Python 3.11 (64-bit)) + Scripts\python.exe + Scripts\pythonw.exe + PYTHONPATH + X64 + + + + + + + + + + \ No newline at end of file diff --git a/tests/Python.ExampleApplication.Tests/README.md b/tests/Python.ExampleApplication.Tests/README.md new file mode 100644 index 00000000..d51ccfcb --- /dev/null +++ b/tests/Python.ExampleApplication.Tests/README.md @@ -0,0 +1,11 @@ +# Intro + +This is the test project for the Python Example Application we reference in code samples for documentation. + +## Instructions to run tests + +Refer to [contributing.md](../../src/Python/CONTRIBUTING.md#first-steps) for instructions to install Python and Hatch(optional). Then, + +- Switch to this directory +- Run `python -m pip install -r .\requirements.txt` +- If using `pytest`, simply run `pytest`. If using hatch, run `hatch run test:test` diff --git a/tests/Python.ExampleApplication.Tests/pyproject.toml b/tests/Python.ExampleApplication.Tests/pyproject.toml new file mode 100644 index 00000000..994186af --- /dev/null +++ b/tests/Python.ExampleApplication.Tests/pyproject.toml @@ -0,0 +1,35 @@ +[project] +name = "tableau_migration_example_app_tests" +dynamic = ["version"] + +authors = [ + { name="Salesforce, Inc." }, +] +description = "Tableau Migration SDK - Example Application Tests" + +classifiers = [ + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", +] + +dependencies = [ + "tableau_migration", + "pytest>=8.2.2" +] + +[tool.ruff] +# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default. +select = ["E", "F"] +ignore = ["E501"] + +[tool.hatch.envs.test] +dev-mode = false +dependencies = [ + "pytest>=8.2.2" +] + +[tool.hatch.envs.test.scripts] +test = "pytest" + +[[tool.hatch.envs.test.matrix]] +python = ["3.8", "3.9", "3.10", "3.11", "3.12"] \ No newline at end of file diff --git a/tests/Python.ExampleApplication.Tests/pytest.ini b/tests/Python.ExampleApplication.Tests/pytest.ini new file mode 100644 index 00000000..b5220b51 --- /dev/null +++ b/tests/Python.ExampleApplication.Tests/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +testpaths = + tests + +pythonpath = + ..\..\examples\Python.ExampleApplication \ No newline at end of file diff --git a/tests/Python.ExampleApplication.Tests/requirements.txt b/tests/Python.ExampleApplication.Tests/requirements.txt new file mode 100644 index 00000000..652bc81a --- /dev/null +++ b/tests/Python.ExampleApplication.Tests/requirements.txt @@ -0,0 +1,2 @@ +tableau_migration +pytest>=8.2.2 diff --git a/tests/Python.ExampleApplication.Tests/tests/__init__.py b/tests/Python.ExampleApplication.Tests/tests/__init__.py new file mode 100644 index 00000000..b7e800da --- /dev/null +++ b/tests/Python.ExampleApplication.Tests/tests/__init__.py @@ -0,0 +1,61 @@ +# Copyright (c) 2024, Salesforce, Inc. +# SPDX-License-Identifier: Apache-2 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Testing module for the Python.TestApplication.""" + +import sys +import os +from os.path import abspath +from pathlib import Path +import tableau_migration + +print("Adding example application paths to sys.path") +sys.path.append(abspath(Path(__file__).parent.resolve().__str__() + "/../../../examples/Python.ExampleApplication/")) +sys.path.append(abspath(Path(__file__).parent.resolve().__str__() + "/../../../examples/Python.ExampleApplication/hooks/batch_migration_completed")) +sys.path.append(abspath(Path(__file__).parent.resolve().__str__() + "/../../../examples/Python.ExampleApplication/hooks/filters")) +sys.path.append(abspath(Path(__file__).parent.resolve().__str__() + "/../../../examples/Python.ExampleApplication/hooks/mappings")) +sys.path.append(abspath(Path(__file__).parent.resolve().__str__() + "/../../../examples/Python.ExampleApplication/hooks/migration_action_completed")) +sys.path.append(abspath(Path(__file__).parent.resolve().__str__() + "/../../../examples/Python.ExampleApplication/hooks/post_publish")) +sys.path.append(abspath(Path(__file__).parent.resolve().__str__() + "/../../../examples/Python.ExampleApplication/hooks/transformers")) + + +if os.environ.get('MIG_SDK_PYTHON_BUILD', 'false').lower() == 'true': + print("MIG_SDK_PYTHON_BUILD set to true. Building dotnet binaries for python tests.") + # Not required for GitHub Actions + import subprocess + import shutil + _bin_path = abspath(Path(__file__).parent.resolve().__str__() + "/../../../src/Python/src/tableau_migration/bin") + sys.path.append(_bin_path) + + shutil.rmtree(_bin_path, True) + print("Building required binaries") + _build_script = abspath(Path(__file__).parent.resolve().__str__() + "/../../../src/Python/scripts/build-package.ps1") + subprocess.run(["pwsh", "-c", _build_script, "-Fast", "-IncludeTests"]) +else: + print("MIG_SDK_PYTHON_BUILD set to false. Skipping dotnet build for python tests.") + +print("Adding test helpers to sys.path") +_autofixture_helper_path = abspath(Path(__file__).parent.resolve().__str__() + "/../../../src/Python/tests/helpers") +sys.path.append(_autofixture_helper_path) + +from tableau_migration import clr +clr.AddReference("AutoFixture") +clr.AddReference("AutoFixture.AutoMoq") +clr.AddReference("Moq") +clr.AddReference("Tableau.Migration.Tests") +clr.AddReference("Tableau.Migration") + + + diff --git a/tests/Python.ExampleApplication.Tests/tests/test_default_project_filter.py b/tests/Python.ExampleApplication.Tests/tests/test_default_project_filter.py new file mode 100644 index 00000000..77dd8b63 --- /dev/null +++ b/tests/Python.ExampleApplication.Tests/tests/test_default_project_filter.py @@ -0,0 +1,41 @@ +# Copyright (c) 2024, Salesforce, Inc. +# SPDX-License-Identifier: Apache-2 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from autofixture import AutoFixtureTestBase + +from default_project_filter import DefaultProjectFilter + +from tableau_migration import ContentMigrationItem +from tableau_migration import IProject + +from Tableau.Migration.Content import IProject as DotnetIProject +from Tableau.Migration.Engine import ContentMigrationItem as DotnetContentMigrationItem + +class TestDefaultProjectFilter(AutoFixtureTestBase): + def test_init(self): + DefaultProjectFilter() + + def test_should_migrate(self): + + dotnet_item = self.create(DotnetContentMigrationItem[DotnetIProject]) + item = ContentMigrationItem[IProject](dotnet_item) + + filter = DefaultProjectFilter() + result = filter.should_migrate(item) + + assert item.source_item.name !='Default' + assert result == True + + \ No newline at end of file diff --git a/tests/Python.ExampleApplication.Tests/tests/test_log_migration_batches_hook.py b/tests/Python.ExampleApplication.Tests/tests/test_log_migration_batches_hook.py new file mode 100644 index 00000000..12da4aaf --- /dev/null +++ b/tests/Python.ExampleApplication.Tests/tests/test_log_migration_batches_hook.py @@ -0,0 +1,24 @@ +# Copyright (c) 2024, Salesforce, Inc. +# SPDX-License-Identifier: Apache-2 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from log_migration_batches_hook import LogMigrationBatchesHookForUsers + +class TestLogMigrationBatchesHookForUsers(): + def test_init(self): + LogMigrationBatchesHookForUsers() + + def test_execute(self): + hook = LogMigrationBatchesHookForUsers(); + assert hook._content_type == "User" diff --git a/tests/Tableau.Migration.Tests/AutoFixtureTestBaseTests.cs b/tests/Tableau.Migration.Tests/AutoFixtureTestBaseTests.cs new file mode 100644 index 00000000..d7387703 --- /dev/null +++ b/tests/Tableau.Migration.Tests/AutoFixtureTestBaseTests.cs @@ -0,0 +1,72 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Linq; +using Tableau.Migration.JsonConverters.Exceptions; +using Xunit; + +namespace Tableau.Migration.Tests +{ + public class AutoFixtureTestBaseTests : AutoFixtureTestBase + { + [Fact] + public void Verify_CreateErrors_create_all_exceptions() + { + // Create all the exceptions + var errors = FixtureFactory.CreateErrors(AutoFixture); + + var loadedAssemblies = AppDomain.CurrentDomain.GetAssemblies(); + + var tableauMigrationAssembly = loadedAssemblies.Where(a => a.ManifestModule.Name == "Tableau.Migration.dll").First(); + + // Find all the exception types in the Tableau.Migration assembly + var exceptionTypes = tableauMigrationAssembly.GetTypes() + .Where(t => t.BaseType == typeof(Exception)) + .Where(t => t != typeof(MismatchException)) + .ToList(); + + // Assert that all the types in exceptionTypes exist in the errors list + Assert.True(exceptionTypes.All(t => errors.Any(e => e.GetType() == t))); + } + + [Fact] + public void Verify_CreateErrors_nullable_properties_not_null() + { + string[] ignoredPropertyNames = new string[] { "InnerException" }; + + // Call CreateErrors + var errors = FixtureFactory.CreateErrors(AutoFixture); + + // Verify that every property in all the objects is not null + foreach (var error in errors) + { + var properties = error.GetType().GetProperties() + .Where(prop => !ignoredPropertyNames.Contains(prop.Name)) + .ToArray(); + + Assert.All(properties, (prop) => Assert.NotNull(prop)); + + foreach (var property in properties) + { + var value = property.GetValue(error); + Assert.NotNull(value); + } + } + } + } +} diff --git a/tests/Tableau.Migration.Tests/ExceptionComparerTests.cs b/tests/Tableau.Migration.Tests/ExceptionComparerTests.cs new file mode 100644 index 00000000..fb46e5a8 --- /dev/null +++ b/tests/Tableau.Migration.Tests/ExceptionComparerTests.cs @@ -0,0 +1,123 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using Xunit; +using Tableau.Migration; + +namespace Tableau.Migration.Tests.Unit +{ + public class ExceptionComparerTests + { + // Custom exception class for testing IEquatable + private class EquatableException : Exception, IEquatable + { + public EquatableException(string message) : base(message) + { } + + public bool Equals(EquatableException? other) + { + return other != null && Message == other.Message; + } + + public override int GetHashCode() + { + return base.GetHashCode(); + } + } + + private readonly ExceptionComparer _comparer = new ExceptionComparer(); + + [Fact] + public void TestEquals_BothNull_ReturnsTrue() + { + Assert.True(_comparer.Equals(null, null)); + } + + [Fact] + public void TestEquals_OneNull_ReturnsFalse() + { + var ex = new Exception(); + Assert.False(_comparer.Equals(null, ex)); + Assert.False(_comparer.Equals(ex, null)); + } + + [Fact] + public void TestEquals_DifferentTypes_ReturnsFalse() + { + var ex1 = new Exception(); + var ex2 = new InvalidOperationException(); + Assert.False(_comparer.Equals(ex1, ex2)); + } + + [Fact] + public void TestEquals_SameTypeDifferentMessages_ReturnsFalse() + { + var ex1 = new Exception("Message 1"); + var ex2 = new Exception("Message 2"); + Assert.False(_comparer.Equals(ex1, ex2)); + } + + [Fact] + public void TestEquals_SameTypeSameMessage_ReturnsTrue() + { + var ex1 = new Exception("Message"); + var ex2 = new Exception("Message"); + Assert.True(_comparer.Equals(ex1, ex2)); + } + + [Fact] + public void TestEquals_ImplementsIEquatable_ReturnsTrue() + { + var ex1 = new EquatableException("Message"); + var ex2 = new EquatableException("Message"); + Assert.True(_comparer.Equals(ex1, ex2)); + } + + [Fact] + public void TestEquals_ExceptionVsEquatableException_ReturnsFalse() + { + var standardException = new Exception("Message"); + var equatableException = new EquatableException("Message"); + Assert.False(_comparer.Equals(standardException, equatableException)); + } + + + [Fact] + public void TestGetHashCode_DifferentMessages_DifferentHashCodes() + { + var ex1 = new Exception("Message 1"); + var ex2 = new Exception("Message 2"); + Assert.NotEqual(_comparer.GetHashCode(ex1), _comparer.GetHashCode(ex2)); + } + + [Fact] + public void TestGetHashCode_SameMessage_SameHashCode() + { + var ex1 = new Exception("Message"); + var ex2 = new Exception("Message"); + Assert.Equal(_comparer.GetHashCode(ex1), _comparer.GetHashCode(ex2)); + } + + [Fact] + public void TestGetHashCode_ImplementsIEquatable_ConsistentHashCode() + { + var ex = new EquatableException("Message"); + Assert.Equal(ex.GetHashCode(), _comparer.GetHashCode(ex)); + } + } +} diff --git a/tests/Tableau.Migration.Tests/Unit/Engine/Manifest/TestMigrationManifestSerializer.cs b/tests/Tableau.Migration.Tests/Unit/Engine/Manifest/TestMigrationManifestSerializer.cs new file mode 100644 index 00000000..e5cd3192 --- /dev/null +++ b/tests/Tableau.Migration.Tests/Unit/Engine/Manifest/TestMigrationManifestSerializer.cs @@ -0,0 +1,95 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.IO; +using System.IO.Abstractions; +using System.IO.Abstractions.TestingHelpers; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using AutoFixture; +using Microsoft.Extensions.Logging; +using Moq; +using Tableau.Migration.Engine.Manifest; +using Tableau.Migration.Resources; +using Xunit; + +namespace Tableau.Migration.Tests.Unit.Engine.Manifest +{ + public class TestMigrationManifestSerializer : AutoFixtureTestBase + { + // If you need to debug these tests and you need access to the file that is saved and loaded, + // you need to make some temporary changes to this file. + // + // The TestMigrationManifestSerializer ctor creates a MockFileSystem, so files are not actually + // saved to disk. If you want to see the manifest that is created, change MockFileSystem line to + // AutoFixture.Register(() => new FileSystem()); + // That will mean the test will use the real file system. + // + // The actual tests also a the temp file to save the manifest to. You can change that to a real filepath + // so it's easier to find during the actual debugging. + + public TestMigrationManifestSerializer() + { + AutoFixture.Register(() => new MockFileSystem()); + } + + [Fact] + public async Task ManifestSaveLoadAsync() + { + // Arrange + var manifest = Create(); + + var tempFile = Path.GetTempFileName(); + + // Verify that the test infra is working + Assert.True(manifest.Entries.Any()); + Assert.True(manifest.Errors.Any()); + + var serializer = Create(); + var cancel = new CancellationToken(); + + // Act + await serializer.SaveAsync(manifest, tempFile); + var loadedManifest = await serializer.LoadAsync(tempFile, cancel); + + // Assert + Assert.NotNull(loadedManifest); + Assert.Equal(manifest as MigrationManifest, loadedManifest); + } + + [Fact] + public async Task ManifestSaveLoad_DifferentVersionAsync() + { + var localizer = Create(); + var logFactory = Create(); + + var mockManifest = new Mock(localizer, logFactory, Guid.NewGuid(), Guid.NewGuid(), null) { CallBase = true }; + mockManifest.Setup(m => m.ManifestVersion).Returns(1); + + var serializer = Create(); + var cancel = new CancellationToken(); + + // Save manifest V2, then try to load with the MigrationManifestSerializer that only supports V1 + var tempFile = Path.GetTempFileName(); + await serializer.SaveAsync(mockManifest.Object, tempFile); + + await Assert.ThrowsAsync(() => serializer.LoadAsync(tempFile, cancel)); + } + } +} diff --git a/tests/Tableau.Migration.Tests/Unit/JsonConverter/SerializableObjects/TestSerializableContentLocation.cs b/tests/Tableau.Migration.Tests/Unit/JsonConverter/SerializableObjects/TestSerializableContentLocation.cs new file mode 100644 index 00000000..d55b4d10 --- /dev/null +++ b/tests/Tableau.Migration.Tests/Unit/JsonConverter/SerializableObjects/TestSerializableContentLocation.cs @@ -0,0 +1,104 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using Tableau.Migration.JsonConverters.Exceptions; +using Tableau.Migration.JsonConverters.SerializableObjects; +using Xunit; + +namespace Tableau.Migration.Tests.Unit.JsonConverter.SerializableObjects +{ + public class TestSerializableContentLocation : AutoFixtureTestBase + { + [Fact] + public void AsContentLocation() + { + var input = Create(); + + var contentLocation = input.AsContentLocation(); + + Assert.Equal(input.Path, contentLocation.Path); + Assert.Equal(input.PathSegments, contentLocation.PathSegments); + Assert.Equal(input.PathSeparator, contentLocation.PathSeparator); + Assert.Equal(input.IsEmpty, contentLocation.IsEmpty); + Assert.Equal(input.Name, contentLocation.Name); + } + + [Fact] + public void BadDeserialization_NullPath() + { + var input = Create(); + input.Path = null; + + Assert.Throws(() => input.AsContentLocation()); + } + + [Fact] + public void BadDeserialization_NullPathSegments() + { + var input = Create(); + input.PathSegments = null; + + Assert.Throws(() => input.AsContentLocation()); + } + + [Fact] + public void BadDeserialization_NullPathSeperator() + { + var input = Create(); + input.PathSeparator = null; + + Assert.Throws(() => input.AsContentLocation()); + } + + [Fact] + public void BadDeserialization_NullName() + { + var input = Create(); + input.Name = null; + + Assert.Throws(() => input.AsContentLocation()); + } + + [Fact] + public void BadDeserialization_PathDoesNotMatchSegments() + { + var input = Create(); + input.Path = "Path"; + + Assert.Throws(() => input.AsContentLocation()); + } + + [Fact] + public void BadDeserialization_NameDoesNotMatchSegments() + { + var input = Create(); + input.Name = "Name"; + + Assert.Throws(() => input.AsContentLocation()); + } + + [Fact] + public void BadDeserialization_IsEmptyDoesNotMatchSegments() + { + var input = Create(); + input.IsEmpty = !input.IsEmpty; + + Assert.Throws(() => input.AsContentLocation()); + } + } +} diff --git a/tests/Tableau.Migration.Tests/Unit/JsonConverter/SerializableObjects/TestSerializableContentReference.cs b/tests/Tableau.Migration.Tests/Unit/JsonConverter/SerializableObjects/TestSerializableContentReference.cs new file mode 100644 index 00000000..1ebb214a --- /dev/null +++ b/tests/Tableau.Migration.Tests/Unit/JsonConverter/SerializableObjects/TestSerializableContentReference.cs @@ -0,0 +1,78 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + + +using System; +using Tableau.Migration.JsonConverters.SerializableObjects; +using Xunit; + +namespace Tableau.Migration.Tests.Unit.JsonConverter.SerializableObjects +{ + public class TestSerializableContentReference : AutoFixtureTestBase + { + [Fact] + public void AsContentReferenceStub() + { + var input = Create(); + + var output = input.AsContentReferenceStub(); + + Assert.NotNull(output); + + Assert.Equal(Guid.Parse(input.Id!), output.Id); + Assert.Equal(input.ContentUrl, output.ContentUrl); + Assert.Equal(input.Location!.AsContentLocation(), output.Location); + Assert.Equal(input.Name, output.Name); + } + + [Fact] + public void BadDeserialization_NullId() + { + var input = Create(); + input.Id = null; + + Assert.Throws(() => input.AsContentReferenceStub()); + } + + [Fact] + public void BadDeserialization_NullContentUrl() + { + var input = Create(); + input.ContentUrl = null; + + Assert.Throws(() => input.AsContentReferenceStub()); + } + + [Fact] + public void BadDeserialization_NullLocation() + { + var input = Create(); + input.Location = null; + + Assert.Throws(() => input.AsContentReferenceStub()); + } + + [Fact] + public void BadDeserialization_NullName() + { + var input = Create(); + input.Name = null; + + Assert.Throws(() => input.AsContentReferenceStub()); + } + } +} diff --git a/tests/Tableau.Migration.Tests/Unit/JsonConverter/SerializableObjects/TestSerializableManifestEntry.cs b/tests/Tableau.Migration.Tests/Unit/JsonConverter/SerializableObjects/TestSerializableManifestEntry.cs new file mode 100644 index 00000000..83600d83 --- /dev/null +++ b/tests/Tableau.Migration.Tests/Unit/JsonConverter/SerializableObjects/TestSerializableManifestEntry.cs @@ -0,0 +1,85 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Linq; +using Moq; +using Tableau.Migration.Engine.Manifest; +using Tableau.Migration.JsonConverters.SerializableObjects; +using Xunit; + +namespace Tableau.Migration.Tests.Unit.JsonConverter.SerializableObjects +{ + public class TestSerializableManifestEntry : AutoFixtureTestBase + { + private readonly Mock mockPartition = new(); + + [Fact] + public void AsMigrationManifestEntryWithDestination() + { + var input = Create(); + + var output = input.AsMigrationManifestEntry(mockPartition.Object); + + Assert.NotNull(output); + + Assert.Equal(input.Source!.AsContentReferenceStub(), output.Source); + Assert.Equal(input.MappedLocation!.AsContentLocation(), output.MappedLocation); + Assert.Equal((int)input.Status, (int)output.Status); + Assert.Equal(input.HasMigrated, output.HasMigrated); + Assert.Equal(input.Destination?.AsContentReferenceStub(), output.Destination); + Assert.Equal(input?.Errors?.Select(e => e.Error), output.Errors); + } + + [Fact] + public void AsMigrationManifestEntryWithoutDestination() + { + var input = Create(); + input.Destination = null; + input.MappedLocation = input.Source!.Location; + + var output = input.AsMigrationManifestEntry(mockPartition.Object); + + Assert.NotNull(output); + + Assert.Equal(input.Source!.AsContentReferenceStub(), output.Source); + Assert.Null(output.Destination); + Assert.Equal(input.MappedLocation!.AsContentLocation(), output.MappedLocation); + Assert.Equal((int)input.Status, (int)output.Status); + Assert.Equal(input.HasMigrated, output.HasMigrated); + Assert.Equal(input?.Errors?.Select(e => e.Error), output.Errors); + } + + [Fact] + public void BadDeserialization_NullSource() + { + var input = Create(); + input.Source = null; + + Assert.Throws(() => input.AsMigrationManifestEntry(mockPartition.Object)); + } + + [Fact] + public void BadDeserialization_NullMappedLocation() + { + var input = Create(); + input.MappedLocation = null; + + Assert.Throws(() => input.AsMigrationManifestEntry(mockPartition.Object)); + } + } +} diff --git a/tests/Tableau.Migration.Tests/Unit/JsonConverter/SerializedExceptionJsonConverterTests.cs b/tests/Tableau.Migration.Tests/Unit/JsonConverter/SerializedExceptionJsonConverterTests.cs new file mode 100644 index 00000000..e37701b1 --- /dev/null +++ b/tests/Tableau.Migration.Tests/Unit/JsonConverter/SerializedExceptionJsonConverterTests.cs @@ -0,0 +1,122 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Text.Json; +using Tableau.Migration.Engine.Manifest; +using Tableau.Migration.JsonConverters; +using Tableau.Migration.JsonConverters.SerializableObjects; +using Xunit; + +namespace Tableau.Migration.Tests.Unit.JsonConverter +{ + public class SerializedExceptionJsonConverterTests : AutoFixtureTestBase + { + private readonly JsonSerializerOptions _serializerOptions; + + public SerializedExceptionJsonConverterTests() + { + _serializerOptions = new JsonSerializerOptions() + { + WriteIndented = true, + }; + + foreach (var converter in MigrationManifestSerializer.CreateConverters()) + { + _serializerOptions.Converters.Add(converter); + } + } + + [Theory] + [ClassData(typeof(SerializableExceptionTypeData))] + public void WriteAndReadBack_ExceptionObject_SerializesAndDeserializesToJson(SerializableException ex) + { + // Arrange + var converter = new SerializedExceptionJsonConverter(); + + Assert.NotNull(ex.Error); + var exceptionNamespace = ex.Error.GetType().Namespace; + Assert.NotNull(exceptionNamespace); + + if (!exceptionNamespace.StartsWith("System")) // Built in Exception is not equatable + { + // We require all custom exception to be equatable so we can test serializability. + Assert.True(ex.Error.ImplementsEquatable()); + } + + // Serialize + using (var memoryStream = new MemoryStream()) + { + var writer = new Utf8JsonWriter(memoryStream); + converter.Write(writer, ex, _serializerOptions); + writer.Flush(); + var json = Encoding.UTF8.GetString(memoryStream.ToArray()); + + var expectedJsonMessage = ex.Error?.Message.Replace("'", "\\u0027") ?? ""; + + // Assert Writer + Assert.NotNull(json); + Assert.NotEmpty(json); + Assert.Contains(expectedJsonMessage, json); + + // Deserialize + var reader = new Utf8JsonReader(Encoding.UTF8.GetBytes(json)); + var result = converter.Read(ref reader, typeof(Exception), _serializerOptions); + + // Assert + Assert.NotNull(result); + Assert.NotNull(result.Error); + Assert.Equal(ex.Error!.Message, result.Error.Message); + + if (!exceptionNamespace.StartsWith("System")) // Built in Exception is not equatable + { + Assert.Equal(ex.Error, result.Error); + } + } + } + + + /// + /// Provides a collection of serializable exception objects. + /// + public class SerializableExceptionTypeData : AutoFixtureTestBase, IEnumerable + { + /// + /// Returns an enumerator that iterates through the collection. + /// + /// An enumerator that can be used to iterate through the collection. + public IEnumerator GetEnumerator() + { + foreach (var e in FixtureFactory.CreateErrors(AutoFixture)) + { + var ex = new SerializableException(e); + yield return new object[] { ex }; + } + } + + /// + /// Returns an enumerator that iterates through the collection. + /// + /// An enumerator that can be used to iterate through the collection. + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + } +}