From d69085067b2261c75f3b49a0e43a4c26aa49d743 Mon Sep 17 00:00:00 2001 From: Marko Lahma Date: Wed, 14 Apr 2021 15:01:50 +0300 Subject: [PATCH] Support skipping output generation based on API specification checksum (#3404) --- .gitignore | 3 +- .../OpenApiDocumentProvider.cs | 4 +- .../Models/CSharpFileTemplateModel.cs | 6 ++ .../Templates/File.liquid | 27 +++---- .../Models/TypeScriptFileTemplateModel.cs | 10 ++- .../Templates/File.liquid | 3 + .../ClientGeneratorBase.cs | 70 +++++++++++++++++++ .../ClientGeneratorBaseSettings.cs | 6 ++ .../CodeGeneratorCommandBase.cs | 7 ++ .../OpenApiToCSharpClientCommand.cs | 6 +- .../OpenApiToTypeScriptClientCommand.cs | 7 +- .../Commands/OutputCommandBase.cs | 14 ++-- src/NSwag.Core/OpenApiDocument.cs | 31 +++++++- .../AspNetCoreOpenApiDocumentGenerator.cs | 12 ++-- 14 files changed, 169 insertions(+), 37 deletions(-) diff --git a/.gitignore b/.gitignore index 3d1e58e2fe..546d4d9dc8 100644 --- a/.gitignore +++ b/.gitignore @@ -74,4 +74,5 @@ src/packages/** /src/NSwag.Console/Properties/launchSettings.json # Ignore files from JetBrainds Rider -/src/.idea/ \ No newline at end of file +/src/.idea/ +_ReSharper.Caches/ diff --git a/src/NSwag.AspNetCore/OpenApiDocumentProvider.cs b/src/NSwag.AspNetCore/OpenApiDocumentProvider.cs index 04e3e0e816..ff5f61164b 100644 --- a/src/NSwag.AspNetCore/OpenApiDocumentProvider.cs +++ b/src/NSwag.AspNetCore/OpenApiDocumentProvider.cs @@ -28,7 +28,7 @@ public OpenApiDocumentProvider(IServiceProvider serviceProvider, IEnumerable GenerateAsync(string documentName) + public Task GenerateAsync(string documentName) { if (documentName == null) { @@ -53,7 +53,7 @@ public async Task GenerateAsync(string documentName) $"Add with the AddSwagger()/AddOpenApi() methods in ConfigureServices()."); } - return await document.Generator.GenerateAsync(_serviceProvider); + return document.Generator.GenerateAsync(_serviceProvider); } // Called by the dotnet-getdocument tool from the Microsoft.Extensions.ApiDescription.Server package. diff --git a/src/NSwag.CodeGeneration.CSharp/Models/CSharpFileTemplateModel.cs b/src/NSwag.CodeGeneration.CSharp/Models/CSharpFileTemplateModel.cs index 751645bd7a..01a938d48e 100644 --- a/src/NSwag.CodeGeneration.CSharp/Models/CSharpFileTemplateModel.cs +++ b/src/NSwag.CodeGeneration.CSharp/Models/CSharpFileTemplateModel.cs @@ -48,8 +48,14 @@ public CSharpFileTemplateModel( _clientCode = clientTypes.Concatenate(); Classes = dtoTypes.Concatenate(); + SourceSha = _settings.ChecksumCacheEnabled ? _document.GetChecksum() : ""; } + /// + /// Gets the checksum for the document that was used to produce the file. + /// + public string SourceSha { get; } + /// Gets the namespace. public string Namespace => _settings.CSharpGeneratorSettings.Namespace ?? string.Empty; diff --git a/src/NSwag.CodeGeneration.CSharp/Templates/File.liquid b/src/NSwag.CodeGeneration.CSharp/Templates/File.liquid index a7643436ca..7f25b184dc 100644 --- a/src/NSwag.CodeGeneration.CSharp/Templates/File.liquid +++ b/src/NSwag.CodeGeneration.CSharp/Templates/File.liquid @@ -1,6 +1,9 @@ //---------------------- // // Generated using the NSwag toolchain v{{ ToolchainVersion }} (http://NSwag.org) +{% if SourceSha != "" -%} +// SourceSHA: {{ SourceSha }} +{% endif -%} // //---------------------- @@ -22,7 +25,7 @@ using {{ usage }}; namespace {{ Namespace }} { using System = global::System; - + {{ Clients | tab }} {% if GenerateContracts -%} @@ -100,14 +103,14 @@ namespace {{ Namespace }} public FileResponse(int statusCode, System.Collections.Generic.IReadOnlyDictionary> headers, System.IO.Stream stream, System.IDisposable client, System.IDisposable response) {% endif -%} { - StatusCode = statusCode; - Headers = headers; - Stream = stream; - _client = client; + StatusCode = statusCode; + Headers = headers; + Stream = stream; + _client = client; _response = response; } - public void Dispose() + public void Dispose() { Stream.Dispose(); if (_response != null) @@ -126,10 +129,10 @@ namespace {{ Namespace }} public int StatusCode { get; private set; } public System.Collections.Generic.IReadOnlyDictionary> Headers { get; private set; } - - public {{ responseClassName }}(int statusCode, System.Collections.Generic.IReadOnlyDictionary> headers) + + public {{ responseClassName }}(int statusCode, System.Collections.Generic.IReadOnlyDictionary> headers) { - StatusCode = statusCode; + StatusCode = statusCode; Headers = headers; } } @@ -138,8 +141,8 @@ namespace {{ Namespace }} public partial class {{ responseClassName }} : {{ responseClassName }} { public TResult Result { get; private set; } - - public {{ responseClassName }}(int statusCode, System.Collections.Generic.IReadOnlyDictionary> headers, TResult result) + + public {{ responseClassName }}(int statusCode, System.Collections.Generic.IReadOnlyDictionary> headers, TResult result) : base(statusCode, headers) { Result = result; @@ -171,7 +174,7 @@ namespace {{ Namespace }} : base(message + "\n\nStatus: " + statusCode + "\nResponse: \n" + ((response == null) ? "(null)" : response.Substring(0, response.Length >= 512 ? 512 : response.Length)), innerException) { StatusCode = statusCode; - Response = response; + Response = response; Headers = headers; } diff --git a/src/NSwag.CodeGeneration.TypeScript/Models/TypeScriptFileTemplateModel.cs b/src/NSwag.CodeGeneration.TypeScript/Models/TypeScriptFileTemplateModel.cs index 3bef7e21af..3ca33da08d 100644 --- a/src/NSwag.CodeGeneration.TypeScript/Models/TypeScriptFileTemplateModel.cs +++ b/src/NSwag.CodeGeneration.TypeScript/Models/TypeScriptFileTemplateModel.cs @@ -49,8 +49,14 @@ public TypeScriptFileTemplateModel( Types = dtoTypes.OrderByBaseDependency().Concatenate(); ExtensionCodeBottom = GenerateExtensionCodeAfter(); Framework = new TypeScriptFrameworkModel(settings); + SourceSha = _settings.ChecksumCacheEnabled ? _document.GetChecksum() : ""; } + /// + /// Gets the checksum for the document that was used to produce the file. + /// + public string SourceSha { get; } + /// Gets framework specific information. public TypeScriptFrameworkModel Framework { get; set; } @@ -127,8 +133,8 @@ public IEnumerable ResponseClassNames public bool RequiresFileParameterInterface => !_settings.TypeScriptGeneratorSettings.ExcludedTypeNames.Contains("FileParameter") && (_document.Operations.Any(o => o.Operation.ActualParameters.Any(p => p.ActualTypeSchema.IsBinary)) || - _document.Operations.Any(o => o.Operation?.RequestBody?.Content?.Any(c => c.Value.Schema?.IsBinary == true || - c.Value.Schema?.ActualProperties.Any(p => p.Value.IsBinary || + _document.Operations.Any(o => o.Operation?.RequestBody?.Content?.Any(c => c.Value.Schema?.IsBinary == true || + c.Value.Schema?.ActualProperties.Any(p => p.Value.IsBinary || p.Value.Item?.IsBinary == true || p.Value.Items.Any(i => i.IsBinary) ) == true) == true)); diff --git a/src/NSwag.CodeGeneration.TypeScript/Templates/File.liquid b/src/NSwag.CodeGeneration.TypeScript/Templates/File.liquid index c6c0261efc..360284a960 100644 --- a/src/NSwag.CodeGeneration.TypeScript/Templates/File.liquid +++ b/src/NSwag.CodeGeneration.TypeScript/Templates/File.liquid @@ -3,6 +3,9 @@ //---------------------- // // Generated using the NSwag toolchain v{{ ToolchainVersion }} (http://NSwag.org) +{% if SourceSha != "" -%} +// SourceSHA: {{ SourceSha }} +{% endif -%} // //---------------------- // ReSharper disable InconsistentNaming diff --git a/src/NSwag.CodeGeneration/ClientGeneratorBase.cs b/src/NSwag.CodeGeneration/ClientGeneratorBase.cs index 27dc464403..1bdfdc75a5 100644 --- a/src/NSwag.CodeGeneration/ClientGeneratorBase.cs +++ b/src/NSwag.CodeGeneration/ClientGeneratorBase.cs @@ -7,6 +7,7 @@ //----------------------------------------------------------------------- using System.Collections.Generic; +using System.IO; using System.Linq; using NJsonSchema; using NJsonSchema.CodeGeneration; @@ -70,6 +71,12 @@ public string GenerateFile() /// The code public string GenerateFile(ClientGeneratorOutputType outputType) { + if (!OutputRequiresRefresh(BaseSettings.OutputFilePath)) + { + // just give back the original file as it hasn't changed + return File.ReadAllText(BaseSettings.OutputFilePath); + } + var clientTypes = GenerateAllClientTypes(); var dtoTypes = BaseSettings.GenerateDtoTypes ? @@ -92,6 +99,69 @@ public string GenerateFile(ClientGeneratorOutputType outputType) .Replace("\n\n\n", "\n\n"); } + private bool OutputRequiresRefresh(string outputFilePath) + { + if (!BaseSettings.ChecksumCacheEnabled || string.IsNullOrWhiteSpace(outputFilePath) || !File.Exists(outputFilePath)) + { + // no point in checking + return true; + } + + var sourceDocumentChecksum = _document.GetChecksum(); + if (string.IsNullOrWhiteSpace(sourceDocumentChecksum)) + { + return true; + } + + // if we can match both source checksum and toolchain, we can presume file hasn't been changed + var checksumMatches = false; + var toolchainVersionMatches = false; + const string checksumMarker = "SourceSHA: "; + const string toolchainVersionMarker = "toolchain v"; + + try + { + using (var reader = new StreamReader(new FileStream(outputFilePath, FileMode.Open, FileAccess.Read, FileShare.Read))) + { + // read multiple lines as we can have headers before version info + var counter = 10; + string line; + while ((line = reader.ReadLine()) != null && counter > 0) + { + counter--; + var checksumStartIndex = line.IndexOf(checksumMarker); + if (checksumStartIndex > -1) + { + var startIndex = checksumStartIndex + checksumMarker.Length; + var fileSourceSha = line.Substring(startIndex).Trim(); + checksumMatches = fileSourceSha == sourceDocumentChecksum; + } + + var toolchainStartIndex = line.IndexOf(toolchainVersionMarker); + if (toolchainStartIndex > -1) + { + var startIndex = toolchainStartIndex + toolchainVersionMarker.Length; + var length = line.IndexOf(" ", startIndex) - startIndex; + var fileToolchainVersion = line.Substring(startIndex, length).Trim(); + toolchainVersionMatches = fileToolchainVersion == OpenApiDocument.ToolchainVersion; + } + + if (checksumMatches && toolchainVersionMatches) + { + // we are good to go with cached version, no need to refresh + return false; + } + } + } + } + catch + { + // no-go + } + + return true; + } + /// Generates the file. /// The client types. /// The DTO types. diff --git a/src/NSwag.CodeGeneration/ClientGeneratorBaseSettings.cs b/src/NSwag.CodeGeneration/ClientGeneratorBaseSettings.cs index e554f45ba9..00e7e29040 100644 --- a/src/NSwag.CodeGeneration/ClientGeneratorBaseSettings.cs +++ b/src/NSwag.CodeGeneration/ClientGeneratorBaseSettings.cs @@ -77,5 +77,11 @@ public string GenerateControllerName(string controllerName) /// Gets or sets the name of the response class (supports the '{controller}' placeholder). public string ResponseClass { get; set; } + + /// Gets or sets the output file name, used to detect changes. + public string OutputFilePath { get; set; } + + /// Gets or sets whether checksum based output caching can be used. + public bool ChecksumCacheEnabled { get; set; } } } \ No newline at end of file diff --git a/src/NSwag.Commands/Commands/CodeGeneration/CodeGeneratorCommandBase.cs b/src/NSwag.Commands/Commands/CodeGeneration/CodeGeneratorCommandBase.cs index 42e003e7ee..9d6bdd8c73 100644 --- a/src/NSwag.Commands/Commands/CodeGeneration/CodeGeneratorCommandBase.cs +++ b/src/NSwag.Commands/Commands/CodeGeneration/CodeGeneratorCommandBase.cs @@ -41,6 +41,13 @@ public string TemplateDirectory [Argument(Name = "EnumNameGeneratorType", IsRequired = false, Description = "The custom IEnumNameGenerator implementation type in the form 'assemblyName:fullTypeName' or 'fullTypeName').")] public string EnumNameGeneratorType { get; set; } + [Argument(Name = nameof(ChecksumCacheEnabled), IsRequired = false, Description = "Whether output generation can be skipped if source document checksum matches existing output (default: false).")] + public bool ChecksumCacheEnabled + { + get { return Settings.ChecksumCacheEnabled; } + set { Settings.ChecksumCacheEnabled = value; } + } + // TODO: Use InitializeCustomTypes method public void InitializeCustomTypes(AssemblyLoader.AssemblyLoader assemblyLoader) { diff --git a/src/NSwag.Commands/Commands/CodeGeneration/OpenApiToCSharpClientCommand.cs b/src/NSwag.Commands/Commands/CodeGeneration/OpenApiToCSharpClientCommand.cs index 9abba35c3d..1d3bdc7609 100644 --- a/src/NSwag.Commands/Commands/CodeGeneration/OpenApiToCSharpClientCommand.cs +++ b/src/NSwag.Commands/Commands/CodeGeneration/OpenApiToCSharpClientCommand.cs @@ -254,9 +254,9 @@ public override async Task RunAsync(CommandLineProcessor processor, ICon return result; } - public async Task> RunAsync() + public Task> RunAsync() { - return await Task.Run(async () => + return Task.Run(async () => { var document = await GetInputSwaggerDocument().ConfigureAwait(false); var clientGenerator = new CSharpClientGenerator(document, Settings); @@ -270,6 +270,8 @@ public async Task> RunAsync() } else { + // when generating single file allow caching + Settings.OutputFilePath = OutputFilePath; return new Dictionary { { OutputFilePath ?? "Full", clientGenerator.GenerateFile(ClientGeneratorOutputType.Full) } diff --git a/src/NSwag.Commands/Commands/CodeGeneration/OpenApiToTypeScriptClientCommand.cs b/src/NSwag.Commands/Commands/CodeGeneration/OpenApiToTypeScriptClientCommand.cs index 9537395393..094b506ef4 100644 --- a/src/NSwag.Commands/Commands/CodeGeneration/OpenApiToTypeScriptClientCommand.cs +++ b/src/NSwag.Commands/Commands/CodeGeneration/OpenApiToTypeScriptClientCommand.cs @@ -270,7 +270,7 @@ public TypeScriptEnumStyle EnumStyle get { return Settings.TypeScriptGeneratorSettings.EnumStyle; } set { Settings.TypeScriptGeneratorSettings.EnumStyle = value; } } - + [Argument(Name = "UseLeafType", IsRequired = false, Description = "Generate leaf types for an object with discriminator (default: false).")] public bool UseLeafType { @@ -394,9 +394,9 @@ public override async Task RunAsync(CommandLineProcessor processor, ICon return code; } - public async Task RunAsync() + public Task RunAsync() { - return await Task.Run(async () => + return Task.Run(async () => { var additionalCode = ExtensionCode ?? string.Empty; if (DynamicApis.FileExists(additionalCode)) @@ -404,6 +404,7 @@ public async Task RunAsync() additionalCode = DynamicApis.FileReadAllText(additionalCode); } + Settings.OutputFilePath = OutputFilePath; Settings.TypeScriptGeneratorSettings.ExtensionCode = additionalCode; var document = await GetInputSwaggerDocument().ConfigureAwait(false); diff --git a/src/NSwag.Commands/Commands/OutputCommandBase.cs b/src/NSwag.Commands/Commands/OutputCommandBase.cs index f5e1a6727c..2549ab4b07 100644 --- a/src/NSwag.Commands/Commands/OutputCommandBase.cs +++ b/src/NSwag.Commands/Commands/OutputCommandBase.cs @@ -25,7 +25,7 @@ public abstract class OutputCommandBase : IOutputCommand public abstract Task RunAsync(CommandLineProcessor processor, IConsoleHost host); - protected async Task ReadSwaggerDocumentAsync(string input) + protected Task ReadSwaggerDocumentAsync(string input) { if (!IsJson(input) && !IsYaml(input)) { @@ -34,11 +34,11 @@ protected async Task ReadSwaggerDocumentAsync(string input) if (input.EndsWith(".yaml", StringComparison.OrdinalIgnoreCase) || input.EndsWith(".yml", StringComparison.OrdinalIgnoreCase)) { - return await OpenApiYamlDocument.FromUrlAsync(input).ConfigureAwait(false); + return OpenApiYamlDocument.FromUrlAsync(input); } else { - return await OpenApiDocument.FromUrlAsync(input).ConfigureAwait(false); + return OpenApiDocument.FromUrlAsync(input); } } else @@ -46,11 +46,11 @@ protected async Task ReadSwaggerDocumentAsync(string input) if (input.EndsWith(".yaml", StringComparison.OrdinalIgnoreCase) || input.EndsWith(".yml", StringComparison.OrdinalIgnoreCase)) { - return await OpenApiYamlDocument.FromFileAsync(input).ConfigureAwait(false); + return OpenApiYamlDocument.FromFileAsync(input); } else { - return await OpenApiDocument.FromFileAsync(input).ConfigureAwait(false); + return OpenApiDocument.FromFileAsync(input); } } } @@ -58,11 +58,11 @@ protected async Task ReadSwaggerDocumentAsync(string input) { if (IsYaml(input)) { - return await OpenApiYamlDocument.FromYamlAsync(input).ConfigureAwait(false); + return OpenApiYamlDocument.FromYamlAsync(input); } else { - return await OpenApiDocument.FromJsonAsync(input).ConfigureAwait(false); + return OpenApiDocument.FromJsonAsync(input); } } } diff --git a/src/NSwag.Core/OpenApiDocument.cs b/src/NSwag.Core/OpenApiDocument.cs index a56128329a..92dddd5340 100644 --- a/src/NSwag.Core/OpenApiDocument.cs +++ b/src/NSwag.Core/OpenApiDocument.cs @@ -9,8 +9,10 @@ using System; using System.Collections.Generic; using System.Collections.ObjectModel; +using System.Diagnostics; using System.Linq; using System.Reflection; +using System.Text; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; @@ -153,7 +155,7 @@ public static Task FromJsonAsync(string data, string documentPa /// The expected schema type which is used when the type cannot be determined. /// The cancellation token. /// The . - public static Task FromJsonAsync(string data, string documentPath, + public static Task FromJsonAsync(string data, string documentPath, SchemaType expectedSchemaType, CancellationToken cancellationToken = default) { return FromJsonAsync(data, documentPath, expectedSchemaType, null, cancellationToken); @@ -166,7 +168,7 @@ public static Task FromJsonAsync(string data, string documentPa /// The JSON reference resolver factory. /// The cancellation token. /// The . - public static async Task FromJsonAsync(string data, string documentPath, SchemaType expectedSchemaType, + public static async Task FromJsonAsync(string data, string documentPath, SchemaType expectedSchemaType, Func referenceResolverFactory, CancellationToken cancellationToken = default) { // For explanation of the regex use https://regexr.com/ and the below unescaped pattern that is without named groups @@ -244,6 +246,31 @@ public IEnumerable Operations } } + /// + /// Calculates checksum for this document based on JSON contents. + /// + /// Will always return null under .NET Standard 1.0. + public string GetChecksum() + { +#if !NETSTANDARD1_0 + var json = ToJson(SchemaType, Formatting.None); + var sb = new StringBuilder(); + using (var hash = System.Security.Cryptography.SHA256.Create()) + { + var result = hash.ComputeHash(Encoding.UTF8.GetBytes(json)); + foreach (var b in result) + { + sb.Append(b.ToString("x2")); + } + } + + return sb.ToString(); +#else + // not supported + return null; +#endif + } + /// Generates missing or non-unique operation IDs. public void GenerateOperationIds() { diff --git a/src/NSwag.Generation.AspNetCore/AspNetCoreOpenApiDocumentGenerator.cs b/src/NSwag.Generation.AspNetCore/AspNetCoreOpenApiDocumentGenerator.cs index 9add3ef7d5..78332c2398 100644 --- a/src/NSwag.Generation.AspNetCore/AspNetCoreOpenApiDocumentGenerator.cs +++ b/src/NSwag.Generation.AspNetCore/AspNetCoreOpenApiDocumentGenerator.cs @@ -43,7 +43,7 @@ public AspNetCoreOpenApiDocumentGenerator(AspNetCoreOpenApiDocumentGeneratorSett /// Generates the with services from the given service provider. /// The service provider. /// The document - public async Task GenerateAsync(object serviceProvider) + public Task GenerateAsync(object serviceProvider) { var typedServiceProvider = (IServiceProvider)serviceProvider; @@ -53,7 +53,7 @@ public async Task GenerateAsync(object serviceProvider) Settings.ApplySettings(settings, mvcOptions.Value); var apiDescriptionGroupCollectionProvider = typedServiceProvider.GetRequiredService(); - return await GenerateAsync(apiDescriptionGroupCollectionProvider.ApiDescriptionGroups); + return GenerateAsync(apiDescriptionGroupCollectionProvider.ApiDescriptionGroups); } /// Loads the from the given service provider. @@ -65,9 +65,9 @@ public static JsonSerializerSettings GetJsonSerializerSettings(IServiceProvider try { #if NET5_0 || NETCOREAPP3_1 || NETCOREAPP3_0 - options = new Func(() => serviceProvider?.GetRequiredService(typeof(IOptions)) as dynamic)(); + options = new Func(() => serviceProvider?.GetRequiredService(typeof(IOptions)))(); #else - options = new Func(() => serviceProvider?.GetRequiredService(typeof(IOptions)) as dynamic)(); + options = new Func(() => serviceProvider?.GetRequiredService(typeof(IOptions)))(); #endif } catch @@ -77,7 +77,7 @@ public static JsonSerializerSettings GetJsonSerializerSettings(IServiceProvider // Try load ASP.NET Core 3 options var optionsAssembly = Assembly.Load(new AssemblyName("Microsoft.AspNetCore.Mvc.NewtonsoftJson")); var optionsType = typeof(IOptions<>).MakeGenericType(optionsAssembly.GetType("Microsoft.AspNetCore.Mvc.MvcNewtonsoftJsonOptions", true)); - options = serviceProvider?.GetService(optionsType) as dynamic; + options = serviceProvider?.GetService(optionsType); } catch { @@ -138,7 +138,7 @@ public async Task GenerateAsync(ApiDescriptionGroupCollection a /// The settings. public static JsonSerializerSettings GetSystemTextJsonSettings(IServiceProvider serviceProvider) { - // If the ASP.NET Core website does not use Newtonsoft.JSON we need to provide a + // If the ASP.NET Core website does not use Newtonsoft.JSON we need to provide a // contract resolver which reflects best the System.Text.Json behavior. // See https://github.com/RicoSuter/NSwag/issues/2243