[WIP] Integration tests

Chisel.sln

@@ -25,6 +25,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{89268D80-B21
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{AC8C6685-EDF9-443A-BAF6-A5E7CF777B2A}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestApp", "tests\TestApp\TestApp.csproj", "{8B1B3D6A-7100-4DFB-97C9-CF5ACF1A3B08}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -43,10 +45,13 @@ Global
 		{845EDA2A-5207-4C6D-ABE9-9635F4630D90}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{845EDA2A-5207-4C6D-ABE9-9635F4630D90}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{845EDA2A-5207-4C6D-ABE9-9635F4630D90}.Release|Any CPU.Build.0 = Release|Any CPU
+		{8B1B3D6A-7100-4DFB-97C9-CF5ACF1A3B08}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{8B1B3D6A-7100-4DFB-97C9-CF5ACF1A3B08}.Release|Any CPU.ActiveCfg = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{845EDA2A-5207-4C6D-ABE9-9635F4630D90} = {0CC84E67-19D2-480B-B36A-6BB15A9109E7}
 		{F4CAEC64-3B0C-4ACD-BF87-760A838A5D86} = {89268D80-B21D-4C76-AF7F-796AAD1E00D9}
 		{EC3CCB92-1AA1-4C33-B296-D7111EEF84E4} = {AC8C6685-EDF9-443A-BAF6-A5E7CF777B2A}
+		{8B1B3D6A-7100-4DFB-97C9-CF5ACF1A3B08} = {AC8C6685-EDF9-443A-BAF6-A5E7CF777B2A}
 	EndGlobalSection
 EndGlobal

tests/Chisel.Tests/Chisel.Tests.csproj

@@ -7,6 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="CliWrap" Version="3.6.6" />
     <PackageReference Include="FluentAssertions" Version="6.12.0" />
     <PackageReference Include="GitHubActionsTestLogger" Version="2.3.3" PrivateAssets="all" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.9.0" />
@@ -18,6 +19,9 @@
   </ItemGroup>
 
   <ItemGroup>
+    <Reference Include="NuGet.Frameworks">
+      <HintPath>$([MSBuild]::NormalizePath('$(MSBuildExtensionsPath)', 'NuGet.Frameworks.dll'))</HintPath>
+    </Reference>
     <Reference Include="NuGet.ProjectModel">
       <HintPath>$([MSBuild]::NormalizePath('$(MSBuildExtensionsPath)', 'NuGet.ProjectModel.dll'))</HintPath>
     </Reference>

tests/Chisel.Tests/ChiseledAppTests.cs

@@ -0,0 +1,76 @@
+using System;
+using System.Diagnostics;
+using System.Text;
+using System.Threading.Tasks;
+using CliWrap;
+using CliWrap.Exceptions;
+using FluentAssertions;
+using FluentAssertions.Execution;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Chisel.Tests;
+
+[Trait("Category", "Integration")]
+public sealed class ChiseledAppTests : IDisposable, IClassFixture<TestApp>
+{
+    readonly ITestOutputHelper _outputHelper;
+    readonly TestApp _testApp;
+    readonly AssertionScope _scope;
+
+    public ChiseledAppTests(ITestOutputHelper outputHelper, TestApp testApp)
+    {
+        _outputHelper = outputHelper;
+        _testApp = testApp;
+        _scope = new AssertionScope();
+    }
+
+    public void Dispose()
+    {
+        _scope.Dispose();
+    }
+
+    public static readonly TheoryData<PublishMode> PublishModeData = new(Enum.GetValues<PublishMode>());
+
+    [Theory]
+    [MemberData(nameof(PublishModeData))]
+    public async Task RunTestApp(PublishMode publishMode)
+    {
+        var (stdOut, stdErr) = await RunTestAppAsync(publishMode);
+        stdOut.Should().Contain("✅");
+        stdErr.Should().BeEmpty();
+    }
+
+    async Task<(string StdOut, string StdErr)> RunTestAppAsync(PublishMode publishMode, params string[] args)
+    {
+        var stdOutBuilder = new StringBuilder();
+        var stdErrBuilder = new StringBuilder();
+
+        var command = Cli.Wrap(_testApp.GetExecutablePath(publishMode))
+            .WithArguments(args)
+            .WithValidation(CommandResultValidation.None)
+            .WithStandardOutputPipe(PipeTarget.ToStringBuilder(stdOutBuilder))
+            .WithStandardErrorPipe(PipeTarget.ToStringBuilder(stdErrBuilder));
+
+        _outputHelper.WriteLine(command.ToString());
+
+        var stopwatch = Stopwatch.StartNew();
+        var result = await command.ExecuteAsync();
+        var executionTime = stopwatch.ElapsedMilliseconds;
+
+        var stdOut = stdOutBuilder.ToString().Trim();
+        var stdErr = stdErrBuilder.ToString().Trim();
+
+        _outputHelper.WriteLine($"Executed in {executionTime} ms");
+        _outputHelper.WriteLine(stdOut.Length > 0 ? $"stdout: {stdOut}" : "nothing on stdout");
+        _outputHelper.WriteLine(stdErr.Length > 0 ? $"stderr: {stdErr}" : "nothing on stderr");
+        _outputHelper.WriteLine("");
+
+        if (result.ExitCode != 0)
+        {
+            throw new CommandExecutionException(command, result.ExitCode, $"An unexpected exception has occurred while running {command}{Environment.NewLine}{stdErr}".Trim());
+        }
+
+        return (stdOut, stdErr);
+    }
+}

tests/Chisel.Tests/DirectoryInfoExtensions.cs

@@ -0,0 +1,13 @@
+using System.IO;
+using System.Linq;
+
+namespace Chisel.Tests;
+
+static class DirectoryInfoExtensions
+{
+    public static DirectoryInfo SubDirectory(this DirectoryInfo directory, params string[] paths)
+        => new(Path.GetFullPath(Path.Combine(paths.Prepend(directory.FullName).ToArray())));
+
+    public static FileInfo File(this DirectoryInfo directory, params string[] paths)
+        => new(Path.GetFullPath(Path.Combine(paths.Prepend(directory.FullName).ToArray())));
+}

tests/Chisel.Tests/PublishMode.cs

@@ -0,0 +1,18 @@
+namespace Chisel.Tests;
+
+/// <summary>
+/// The possible application publish modes for the TestApp.
+/// See also the <a href="https://learn.microsoft.com/en-us/dotnet/core/deploying/">.NET application publishing overview</a> documentation.
+/// </summary>
+public enum PublishMode
+{
+    /// <summary>
+    /// Standard app publish, all dlls and related files are copied along the main executable.
+    /// </summary>
+    Standard,
+
+    /// <summary>
+    /// Publish a single file as a framework-dependent binary.
+    /// </summary>
+    SingleFile,
+}

tests/Chisel.Tests/TestApp.cs

@@ -0,0 +1,163 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using System.Runtime.Versioning;
+using System.Text;
+using System.Threading.Tasks;
+using CliWrap;
+using CliWrap.Exceptions;
+using FluentAssertions;
+using NuGet.Frameworks;
+using Xunit;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace Chisel.Tests;
+
+public class TestApp : IAsyncLifetime
+{
+    readonly IMessageSink _messageSink;
+    readonly DirectoryInfo _workingDirectory;
+    readonly Dictionary<PublishMode, FileInfo> _executables;
+
+    public TestApp(IMessageSink messageSink)
+    {
+        _messageSink = messageSink;
+        var targetFrameworkAttribute = typeof(TestApp).Assembly.GetCustomAttribute<TargetFrameworkAttribute>();
+        var framework = NuGetFramework.Parse(targetFrameworkAttribute?.FrameworkName ?? "?unknown-tfm?");
+        _workingDirectory = GetDirectory("tests", $"TestApp-{framework}");
+        _workingDirectory.Create();
+        foreach (var file in GetDirectory("tests", "TestApp").EnumerateFiles())
+        {
+            file.CopyTo(_workingDirectory.File(file.Name).FullName, overwrite: true);
+        }
+        _executables = new Dictionary<PublishMode, FileInfo>();
+    }
+
+    async Task IAsyncLifetime.InitializeAsync()
+    {
+        await CreateTestAppAsync();
+    }
+
+    Task IAsyncLifetime.DisposeAsync()
+    {
+        _workingDirectory.Delete(recursive: true);
+        return Task.CompletedTask;
+    }
+
+    public string GetExecutablePath(PublishMode publishMode) => _executables[publishMode].FullName;
+
+    async Task CreateTestAppAsync()
+    {
+        // It might be tempting to do pack -> restore -> build --no-restore -> publish --no-build (and parallelize over publish modes)
+        // But this would fail because of https://github.com/dotnet/sdk/issues/17526 and probably because of other unforeseen bugs
+        // preventing from running multiple `dotnet publish` commands with different parameters.
+
+        await PackAsync();
+        await RestoreAsync();
+        foreach (var publishMode in Enum.GetValues<PublishMode>())
+        {
+            await PublishAsync(publishMode);
+        }
+    }
+
+    async Task PackAsync()
+    {
+        var projectFile = GetFile("src", "Chisel", "Chisel.csproj");
+        var packArgs = new[] {
+            "pack", projectFile.FullName,
+            "--configuration", "Release",
+            "--output", _workingDirectory.FullName,
+            "-p:MinVerSkip=true",
+            "-p:Version=0.0.0-IntegrationTest.0",
+        };
+        await RunDotnetAsync(_workingDirectory, packArgs);
+    }
+
+    async Task RestoreAsync()
+    {
+        // Can't use "--source . --source https://api.nuget.org/v3/index.json" because of https://github.com/dotnet/sdk/issues/27202 => a nuget.config file is used instead.
+        // It also has the benefit of using settings _only_ from the specified config file, ignoring the global nuget.config where package source mapping could interfere with the local source.
+        var restoreArgs = new[] {
+            "restore",
+            "--configfile", "nuget.config",
+            "-p:Configuration=Release",
+        };
+        await RunDotnetAsync(_workingDirectory, restoreArgs);
+    }
+
+    async Task PublishAsync(PublishMode publishMode)
+    {
+        var publishDirectory = _workingDirectory.SubDirectory("publish");
+
+        var outputDirectory = publishDirectory.SubDirectory(publishMode.ToString());
+
+        var publishArgsBase = new[] {
+            "publish",
+            "--no-restore",
+            "--configuration", "Release",
+            "--output", outputDirectory.FullName,
+            //"-p:SelfContained=false",
+            //$"-p:TargetFramework={TargetFramework}"
+        };
+        var publishSingleFile = $"-p:PublishSingleFile={publishMode is PublishMode.SingleFile}";
+        var publishArgs = publishArgsBase.Append(publishSingleFile).ToArray();
+        await RunDotnetAsync(_workingDirectory, publishArgs);
+
+        var executableFileName = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "TestApp.exe" : "TestApp";
+        var executableFile = new FileInfo(Path.Combine(outputDirectory.FullName, executableFileName));
+        executableFile.Exists.Should().BeTrue();
+        var dlls = executableFile.Directory!.EnumerateFiles("*.dll");
+        if (publishMode == PublishMode.Standard)
+        {
+            dlls.Should().NotBeEmpty(because: $"the test app was _not_ published as single-file ({publishMode})");
+        }
+        else
+        {
+            dlls.Should().BeEmpty(because: $"the test app was published as single-file ({publishMode})");
+            executableFile.Directory.EnumerateFiles().Should().ContainSingle().Which.FullName.Should().Be(executableFile.FullName);
+        }
+
+        _executables[publishMode] = executableFile;
+    }
+
+    async Task RunDotnetAsync(DirectoryInfo workingDirectory, params string[] arguments)
+    {
+        _messageSink.OnMessage(new DiagnosticMessage($"cd {workingDirectory}"));
+        _messageSink.OnMessage(new DiagnosticMessage($"dotnet {string.Join(" ", arguments)}"));
+        var outBuilder = new StringBuilder();
+        var errBuilder = new StringBuilder();
+        var command = Cli.Wrap("dotnet")
+            .WithValidation(CommandResultValidation.None)
+            .WithWorkingDirectory(workingDirectory.FullName)
+            .WithArguments(arguments)
+            .WithStandardOutputPipe(PipeTarget.ToDelegate(line =>
+            {
+                outBuilder.AppendLine(line);
+                _messageSink.OnMessage(new DiagnosticMessage($"==> out: {line}"));
+            }))
+            .WithStandardErrorPipe(PipeTarget.ToDelegate(line =>
+            {
+                errBuilder.AppendLine(line);
+                _messageSink.OnMessage(new DiagnosticMessage($"==> err: {line}"));
+            }));
+
+        var result = await command.ExecuteAsync();
+        if (result.ExitCode != 0)
+        {
+            throw new CommandExecutionException(command, result.ExitCode, $"An unexpected exception has occurred while running {command}{Environment.NewLine}{errBuilder}{outBuilder}".Trim());
+        }
+    }
+
+    static DirectoryInfo GetDirectory(params string[] paths) => new(GetFullPath(paths));
+
+    static FileInfo GetFile(params string[] paths) => new(GetFullPath(paths));
+
+    static string GetFullPath(params string[] paths) => Path.GetFullPath(Path.Combine(new[] { GetThisDirectory(), "..", ".." }.Concat(paths).ToArray()));
+
+    static string GetThisDirectory([CallerFilePath] string path = "") => Path.GetDirectoryName(path)!;
+}

tests/TestApp/Program.cs

@@ -0,0 +1,38 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Text.RegularExpressions;
+using Microsoft.Data.SqlClient;
+
+var dlls = new DirectoryInfo(AppContext.BaseDirectory).EnumerateFiles("*.dll").Where(e => e.Name != "ChiseledSqlClient.dll").OrderByDescending(e => e.Length).ToList();
+if (args.Contains("--dlls") && dlls.Count > 0)
+{
+    var isChiseled = dlls.All(e => !e.Name.Contains("Azure"));
+    var maxLength = dlls.Max(e => e.Name.Length) + (isChiseled ? 0 : 4);
+    Console.WriteLine($"| File {new string(' ', maxLength - 4)}| Size    |");
+    Console.WriteLine($"|---{new string('-', maxLength - 4)}---|---------|");
+    var isAzureDependentRegex = new Regex(@"Azure|Microsoft\.Identity|msal|Web");
+    foreach (var dll in dlls)
+    {
+        var name = !isChiseled && isAzureDependentRegex.IsMatch(dll.Name) ? $"**{dll.Name}**" : dll.Name;
+        Console.WriteLine($"| {name.PadRight(maxLength)} | {dll.Length / 1_000_000.0:F2} MB |");
+    }
+}
+
+var totalDllSize = dlls.Sum(e => e.Length);
+Console.WriteLine($"Total DLL size: {totalDllSize / 1_000_000.0:F1} MB");
+
+var connectionString = args.Length > 0 && !args[^1].StartsWith("--") ? args[^1] : "Server=sqlprosample.database.windows.net;Database=sqlprosample;user=sqlproro;password=nh{Zd?*8ZU@Y}Bb#";
+await using var dataSource = SqlClientFactory.Instance.CreateDataSource(connectionString);
+await using var command = dataSource.CreateCommand("Select @@version");
+try
+{
+    var result = await command.ExecuteScalarAsync();
+    Console.WriteLine($"✅ {result}");
+    return 0;
+}
+catch (Exception exception)
+{
+    Console.Error.WriteLine($"❌ {exception}");
+    return 1;
+}

tests/TestApp/TestApp.csproj

@@ -0,0 +1,31 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <LangVersion>latest</LangVersion>
+    <DebugType>embedded</DebugType>
+    <AutoGenerateBindingRedirects>false</AutoGenerateBindingRedirects>
+    <GenerateSupportedRuntime>false</GenerateSupportedRuntime>
+    <PublishReferencesDocumentationFiles>false</PublishReferencesDocumentationFiles>
+    <AllowedReferenceRelatedFileExtensions>none</AllowedReferenceRelatedFileExtensions>
+    <UseCurrentRuntimeIdentifier>true</UseCurrentRuntimeIdentifier>
+    <SelfContained>false</SelfContained>
+    <SatelliteResourceLanguages>en</SatelliteResourceLanguages>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Chisel" Version="[0.0.0-IntegrationTest.0]" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ChiselPackages Include="Azure.Identity" />
+    <ChiselPackages Include="Microsoft.IdentityModel.JsonWebTokens" />
+    <ChiselPackages Include="Microsoft.IdentityModel.Protocols.OpenIdConnect" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Data.SqlClient" Version="5.2.0" />
+  </ItemGroup>
+
+</Project>

tests/TestApp/nuget.config

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <config>
+    <add key="globalPackagesFolder" value="packages" />
+  </config>
+  <packageSources>
+    <clear />
+    <add key="local" value="." />
+    <add key="nuget" value="https://api.nuget.org/v3/index.json" />
+  </packageSources>
+</configuration>