From 3a99c2174e6752047e8abe20f44397a49d363e7d Mon Sep 17 00:00:00 2001 From: Charlie Poole Date: Sun, 29 Aug 2021 19:29:15 -0700 Subject: [PATCH] Specify console versions for each test separately --- cake/package-tests.cake | 145 ++++++++++++++++++++-------------------- cake/test-reports.cake | 9 ++- 2 files changed, 79 insertions(+), 75 deletions(-) diff --git a/cake/package-tests.cake b/cake/package-tests.cake index 72a5b29..2690600 100644 --- a/cake/package-tests.cake +++ b/cake/package-tests.cake @@ -7,7 +7,7 @@ public class PackageTest { public string Description { get; set; } - public string ConsoleVersion { get; set; } + public string[] TestConsoleVersions { get; set; } public string Arguments { get; set; } public ExpectedResult ExpectedResult { get; set; } } @@ -17,7 +17,7 @@ public class PackageTest ////////////////////////////////////////////////////////////////////// const string DEFAULT_TEST_RESULT_FILE = "TestResult.xml"; -static readonly string[] TEST_CONSOLE_VERSIONS = new[] { "3.12.0", "3.11.1", "3.10.0" }; + public abstract class PackageTester { protected BuildParameters _parameters; @@ -28,60 +28,57 @@ public abstract class PackageTester _parameters = parameters; _context = parameters.Context; - foreach (var consoleVersion in TEST_CONSOLE_VERSIONS) + PackageTests.Add(new PackageTest() { - PackageTests.Add(new PackageTest() + Description = "Project with one assembly, all tests pass", + Arguments = "PassingAssembly.nunit", + TestConsoleVersions = new string[] { "3.12.0", "3.11.1", "3.10.0" }, + ExpectedResult = new ExpectedResult("Passed") { - Description = "Project with one assembly, all tests pass", - Arguments = "PassingAssembly.nunit", - ConsoleVersion = consoleVersion, - ExpectedResult = new ExpectedResult("Passed") - { - Total = 4, - Passed = 4, - Failed = 0, - Warnings = 0, - Inconclusive = 0, - Skipped = 0, - Assemblies = new[] { new ExpectedAssemblyResult("test-lib-1.dll", "net-2.0") } - } - }); - PackageTests.Add(new PackageTest() + Total = 4, + Passed = 4, + Failed = 0, + Warnings = 0, + Inconclusive = 0, + Skipped = 0, + Assemblies = new[] { new ExpectedAssemblyResult("test-lib-1.dll", "net-2.0") } + } + }); + PackageTests.Add(new PackageTest() + { + Description = "Project with one assembly, some failures", + Arguments = "FailingAssembly.nunit", + TestConsoleVersions = new string[] { "3.12.0", "3.11.1", "3.10.0" }, + ExpectedResult = new ExpectedResult("Failed") { - Description = "Project with one assembly, some failures", - Arguments = "FailingAssembly.nunit", - ConsoleVersion = consoleVersion, - ExpectedResult = new ExpectedResult("Failed") - { - Total = 9, - Passed = 4, - Failed = 2, - Warnings = 0, - Inconclusive = 1, - Skipped = 2, - Assemblies = new[] { new ExpectedAssemblyResult("test-lib-2.dll", "net-2.0") } - } - }); - PackageTests.Add(new PackageTest() + Total = 9, + Passed = 4, + Failed = 2, + Warnings = 0, + Inconclusive = 1, + Skipped = 2, + Assemblies = new[] { new ExpectedAssemblyResult("test-lib-2.dll", "net-2.0") } + } + }); + PackageTests.Add(new PackageTest() + { + Description = "Project with both assemblies", + Arguments = "BothAssemblies.nunit", + TestConsoleVersions = new string[] { "3.12.0", "3.11.1", "3.10.0" }, + ExpectedResult = new ExpectedResult("Failed") { - Description = "Project with both assemblies", - Arguments = "BothAssemblies.nunit", - ConsoleVersion = consoleVersion, - ExpectedResult = new ExpectedResult("Failed") - { - Total = 13, - Passed = 8, - Failed = 2, - Warnings = 0, - Inconclusive = 1, - Skipped = 2, - Assemblies = new[] { - new ExpectedAssemblyResult("test-lib-1.dll", "net-2.0"), - new ExpectedAssemblyResult("test-lib-2.dll", "net-2.0") - } + Total = 13, + Passed = 8, + Failed = 2, + Warnings = 0, + Inconclusive = 1, + Skipped = 2, + Assemblies = new[] { + new ExpectedAssemblyResult("test-lib-1.dll", "net-2.0"), + new ExpectedAssemblyResult("test-lib-2.dll", "net-2.0") } - }); - } + } + }); } protected abstract string PackageName { get; } @@ -98,32 +95,36 @@ public abstract class PackageTester foreach (var packageTest in PackageTests) { var resultFile = _parameters.OutputDirectory + DEFAULT_TEST_RESULT_FILE; - // Delete result file ahead of time so we don't mistakenly - // read a left-over file from another test run. Leave the - // file after the run in case we need it to debug a failure. - if (_context.FileExists(resultFile)) - _context.DeleteFile(resultFile); - DisplayBanner(packageTest.Description + " - Console Version " + packageTest.ConsoleVersion); + foreach (var consoleVersion in packageTest.TestConsoleVersions) + { + // Delete result file ahead of time so we don't mistakenly + // read a left-over file from another test run. Leave the + // file after the run in case we need it to debug a failure. + if (_context.FileExists(resultFile)) + _context.DeleteFile(resultFile); - RunConsoleTest(packageTest.ConsoleVersion, packageTest.Arguments); + DisplayBanner(packageTest.Description + " - Console Version " + consoleVersion); - try - { - var result = new ActualResult(_parameters.ProjectDirectory + DEFAULT_TEST_RESULT_FILE); - var report = new PackageTestReport(packageTest, result); - reporter.AddReport(report); + RunConsoleTest(consoleVersion, packageTest.Arguments); - Console.WriteLine(report.Errors.Count == 0 - ? "\nSUCCESS: Test Result matches expected result!" - : "\nERROR: Test Result not as expected!"); - } - catch (Exception ex) - { - reporter.AddReport(new PackageTestReport(packageTest, ex)); + try + { + var result = new ActualResult(_parameters.ProjectDirectory + DEFAULT_TEST_RESULT_FILE); + var report = new PackageTestReport(packageTest, consoleVersion, result); + reporter.AddReport(report); - Console.WriteLine($"\nERROR: No result found."); - Console.WriteLine(ex.ToString()); + Console.WriteLine(report.Errors.Count == 0 + ? "\nSUCCESS: Test Result matches expected result!" + : "\nERROR: Test Result not as expected!"); + } + catch (Exception ex) + { + reporter.AddReport(new PackageTestReport(packageTest, consoleVersion, ex)); + + Console.WriteLine($"\nERROR: No result found."); + Console.WriteLine(ex.ToString()); + } } } diff --git a/cake/test-reports.cake b/cake/test-reports.cake index 9c16474..b152d10 100644 --- a/cake/test-reports.cake +++ b/cake/test-reports.cake @@ -6,13 +6,15 @@ public class PackageTestReport { public PackageTest Test; + public string ConsoleVersion; public ActualResult Result; public List Errors; public List Warnings; - public PackageTestReport(PackageTest test, ActualResult actualResult) + public PackageTestReport(PackageTest test, string consoleVersion, ActualResult actualResult) { Test = test; + ConsoleVersion = consoleVersion; Result = actualResult; Errors = new List(); Warnings = new List(); @@ -55,9 +57,10 @@ public class PackageTestReport Errors.Add($" Found unexpected assembly {actualAssemblies[i].Name}"); } - public PackageTestReport(PackageTest test, Exception ex) + public PackageTestReport(PackageTest test, string consoleVersion, Exception ex) { Test = test; + ConsoleVersion = consoleVersion; Result = null; Errors = new List(); Errors.Add($" {ex.Message}"); @@ -67,7 +70,7 @@ public class PackageTestReport { Console.WriteLine(); Console.WriteLine($"{index}. {Test.Description}"); - Console.WriteLine($" ConsoleVersion: {Test.ConsoleVersion}"); + Console.WriteLine($" ConsoleVersion: {ConsoleVersion}"); Console.WriteLine($" Args: {Test.Arguments}"); Console.WriteLine();