From 5b99712e62367075bbc0b31d232995a3f4acdf08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Mon, 24 Feb 2025 09:27:08 +0100 Subject: [PATCH 01/52] Onboard to new dotnet test experience --- dotnet.config | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 dotnet.config diff --git a/dotnet.config b/dotnet.config new file mode 100644 index 0000000000..63ba841e87 --- /dev/null +++ b/dotnet.config @@ -0,0 +1,2 @@ +[dotnet.test:runner] +name= "MicrosoftTestingPlatform" From af2e1f90d148bdbe91a10e75f51401b67c52f237 Mon Sep 17 00:00:00 2001 From: Youssef Victor Date: Thu, 6 Mar 2025 12:39:31 +0100 Subject: [PATCH 02/52] Update name --- dotnet.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet.config b/dotnet.config index 63ba841e87..1271fc9236 100644 --- a/dotnet.config +++ b/dotnet.config @@ -1,2 +1,2 @@ [dotnet.test:runner] -name= "MicrosoftTestingPlatform" +name= "Microsoft.Testing.Platform" From 39c6c2789ead1b582cac95aee2db90057721a37d Mon Sep 17 00:00:00 2001 From: Youssef Victor Date: Thu, 6 Mar 2025 12:42:05 +0100 Subject: [PATCH 03/52] Revert back --- dotnet.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet.config b/dotnet.config index 1271fc9236..63ba841e87 100644 --- a/dotnet.config +++ b/dotnet.config @@ -1,2 +1,2 @@ [dotnet.test:runner] -name= "Microsoft.Testing.Platform" +name= "MicrosoftTestingPlatform" From 8db05c88f8437e2efd3c2512017e97f8aab91a31 Mon Sep 17 00:00:00 2001 From: Youssef Victor Date: Thu, 6 Mar 2025 12:54:25 +0100 Subject: [PATCH 04/52] Use the old experience for test assets --- .../TempDirectory.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs b/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs index f36305e3de..f564d28b22 100644 --- a/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs +++ b/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs @@ -172,6 +172,16 @@ internal static (string BaseDirectory, string FinalDirectory) CreateUniqueDirect string directoryPath = System.IO.Path.Combine(TestSuiteDirectory, RandomId.Next()); Directory.CreateDirectory(directoryPath); + // Our tests were originally wrote before the enhanced dotnet test support for MTP. + // So, by default we use VSTest. + // We can start to gradually move some of the MTP tests to the new dotnet test experience. + // Note that we have tests that are actually VSTest-specific. + string dotnetConfig = System.IO.Path.Combine(directoryPath, "dotnet.config"); + File.WriteAllText(dotnetConfig, """ + [dotnet.test:runner] + name= "VSTest" + """; + string directoryBuildProps = System.IO.Path.Combine(directoryPath, "Directory.Build.props"); File.WriteAllText(directoryBuildProps, $""" From fe77fea1883b42199bd05bf17c51a7d5c5a083b1 Mon Sep 17 00:00:00 2001 From: Youssef Victor Date: Thu, 6 Mar 2025 13:41:07 +0100 Subject: [PATCH 05/52] Add missing paren --- .../Microsoft.Testing.TestInfrastructure/TempDirectory.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs b/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs index f564d28b22..a3ab00e568 100644 --- a/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs +++ b/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Platform.Helpers; @@ -180,7 +180,7 @@ internal static (string BaseDirectory, string FinalDirectory) CreateUniqueDirect File.WriteAllText(dotnetConfig, """ [dotnet.test:runner] name= "VSTest" - """; + """); string directoryBuildProps = System.IO.Path.Combine(directoryPath, "Directory.Build.props"); File.WriteAllText(directoryBuildProps, $""" From f14c3b217d45bcbee7a3f906d934f756d6dbb3f9 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Thu, 6 Mar 2025 14:28:26 +0100 Subject: [PATCH 06/52] Use TestSuiteDirectory for dotnet.config and make it default working directory when running integration tests --- .../Microsoft.Testing.TestInfrastructure/ProcessFactory.cs | 2 +- .../Microsoft.Testing.TestInfrastructure/TempDirectory.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/Utilities/Microsoft.Testing.TestInfrastructure/ProcessFactory.cs b/test/Utilities/Microsoft.Testing.TestInfrastructure/ProcessFactory.cs index a7a7b0a8fa..da14874ff5 100644 --- a/test/Utilities/Microsoft.Testing.TestInfrastructure/ProcessFactory.cs +++ b/test/Utilities/Microsoft.Testing.TestInfrastructure/ProcessFactory.cs @@ -9,7 +9,7 @@ public static IProcessHandle Start(ProcessConfiguration config, bool cleanDefaul { string fullPath = config.FileName; // Path.GetFullPath(startInfo.FileName); string workingDirectory = config.WorkingDirectory - .OrDefault(Path.GetDirectoryName(config.FileName).OrDefault(Directory.GetCurrentDirectory())); + .OrDefault(TempDirectory.TestSuiteDirectory); ProcessStartInfo processStartInfo = new() { diff --git a/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs b/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs index a3ab00e568..d2b013742c 100644 --- a/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs +++ b/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Platform.Helpers; @@ -176,7 +176,7 @@ internal static (string BaseDirectory, string FinalDirectory) CreateUniqueDirect // So, by default we use VSTest. // We can start to gradually move some of the MTP tests to the new dotnet test experience. // Note that we have tests that are actually VSTest-specific. - string dotnetConfig = System.IO.Path.Combine(directoryPath, "dotnet.config"); + string dotnetConfig = System.IO.Path.Combine(TestSuiteDirectory, "dotnet.config"); File.WriteAllText(dotnetConfig, """ [dotnet.test:runner] name= "VSTest" From 9e2efb34a686644bc95a326a797000a70a0764db Mon Sep 17 00:00:00 2001 From: Youssef Victor Date: Sat, 8 Mar 2025 11:12:28 +0100 Subject: [PATCH 07/52] Update dotnet.config --- dotnet.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet.config b/dotnet.config index 63ba841e87..1271fc9236 100644 --- a/dotnet.config +++ b/dotnet.config @@ -1,2 +1,2 @@ [dotnet.test:runner] -name= "MicrosoftTestingPlatform" +name= "Microsoft.Testing.Platform" From c94ee87ec4be285ab34c402bd40f541282a0db6d Mon Sep 17 00:00:00 2001 From: Youssef Victor Date: Sat, 8 Mar 2025 11:23:43 +0100 Subject: [PATCH 08/52] Revert back again... --- dotnet.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet.config b/dotnet.config index 1271fc9236..63ba841e87 100644 --- a/dotnet.config +++ b/dotnet.config @@ -1,2 +1,2 @@ [dotnet.test:runner] -name= "Microsoft.Testing.Platform" +name= "MicrosoftTestingPlatform" From 91bfcd941bea22fc54bcbb073842ac6661f9e916 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Sat, 8 Mar 2025 13:36:43 +0100 Subject: [PATCH 09/52] Adjust for working directory of testhost --- .../Microsoft.Testing.TestInfrastructure/CommandLine.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Utilities/Microsoft.Testing.TestInfrastructure/CommandLine.cs b/test/Utilities/Microsoft.Testing.TestInfrastructure/CommandLine.cs index 3fd02adb5b..c942c7979d 100644 --- a/test/Utilities/Microsoft.Testing.TestInfrastructure/CommandLine.cs +++ b/test/Utilities/Microsoft.Testing.TestInfrastructure/CommandLine.cs @@ -87,7 +87,7 @@ public async Task RunAsyncAndReturnExitCodeAsync( EnvironmentVariables = environmentVariables, OnErrorOutput = (_, o) => _errorOutputLines.Add(ClearBOM(o)), OnStandardOutput = (_, o) => _standardOutputLines.Add(ClearBOM(o)), - WorkingDirectory = workingDirectory, + WorkingDirectory = workingDirectory ?? Path.GetDirectoryName(command), }; _process = ProcessFactory.Start(startInfo, cleanDefaultEnvironmentVariableIfCustomAreProvided); From ca186fdbaac70f652b32e93e1e7fd465f47d72ca Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Sat, 8 Mar 2025 14:18:12 +0100 Subject: [PATCH 10/52] Avoid racing --- .../TempDirectory.cs | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs b/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs index d2b013742c..d631ad391d 100644 --- a/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs +++ b/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs @@ -146,7 +146,20 @@ public string CopyFile(string filePath) [Obsolete("Don't use directly. Use TestSuiteDirectory property instead.")] private static string GetTestSuiteDirectory() - => System.IO.Path.Combine(RepoRoot, "artifacts", "tmp", Constants.BuildConfiguration, "testsuite"); + { + string testSuiteDirectory = System.IO.Path.Combine(RepoRoot, "artifacts", "tmp", Constants.BuildConfiguration, "testsuite"); + + // Our tests were originally wrote before the enhanced dotnet test support for MTP. + // So, by default we use VSTest. + // We can start to gradually move some of the MTP tests to the new dotnet test experience. + // Note that we have tests that are actually VSTest-specific. + string dotnetConfig = System.IO.Path.Combine(testSuiteDirectory, "dotnet.config"); + File.WriteAllText(dotnetConfig, """ + [dotnet.test:runner] + name= "VSTest" + """); + return testSuiteDirectory; + } [Obsolete("Don't use directly. Use RepoRoot property instead.")] private static string GetRepoRoot() @@ -172,16 +185,6 @@ internal static (string BaseDirectory, string FinalDirectory) CreateUniqueDirect string directoryPath = System.IO.Path.Combine(TestSuiteDirectory, RandomId.Next()); Directory.CreateDirectory(directoryPath); - // Our tests were originally wrote before the enhanced dotnet test support for MTP. - // So, by default we use VSTest. - // We can start to gradually move some of the MTP tests to the new dotnet test experience. - // Note that we have tests that are actually VSTest-specific. - string dotnetConfig = System.IO.Path.Combine(TestSuiteDirectory, "dotnet.config"); - File.WriteAllText(dotnetConfig, """ - [dotnet.test:runner] - name= "VSTest" - """); - string directoryBuildProps = System.IO.Path.Combine(directoryPath, "Directory.Build.props"); File.WriteAllText(directoryBuildProps, $""" From 0bbe49d5a93213b6319e872446f3b8562ab7507b Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Sat, 8 Mar 2025 14:28:15 +0100 Subject: [PATCH 11/52] Create directory --- .../Microsoft.Testing.TestInfrastructure/TempDirectory.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs b/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs index d631ad391d..88295869c5 100644 --- a/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs +++ b/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs @@ -148,6 +148,7 @@ public string CopyFile(string filePath) private static string GetTestSuiteDirectory() { string testSuiteDirectory = System.IO.Path.Combine(RepoRoot, "artifacts", "tmp", Constants.BuildConfiguration, "testsuite"); + Directory.CreateDirectory(testSuiteDirectory); // Our tests were originally wrote before the enhanced dotnet test support for MTP. // So, by default we use VSTest. From 48bea7f286861a9f704b31d08b832bac672fe7aa Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Sat, 8 Mar 2025 14:41:08 +0100 Subject: [PATCH 12/52] Revert --- .../CommandLine.cs | 2 +- .../ProcessFactory.cs | 2 +- .../TempDirectory.cs | 16 +--------------- 3 files changed, 3 insertions(+), 17 deletions(-) diff --git a/test/Utilities/Microsoft.Testing.TestInfrastructure/CommandLine.cs b/test/Utilities/Microsoft.Testing.TestInfrastructure/CommandLine.cs index c942c7979d..3fd02adb5b 100644 --- a/test/Utilities/Microsoft.Testing.TestInfrastructure/CommandLine.cs +++ b/test/Utilities/Microsoft.Testing.TestInfrastructure/CommandLine.cs @@ -87,7 +87,7 @@ public async Task RunAsyncAndReturnExitCodeAsync( EnvironmentVariables = environmentVariables, OnErrorOutput = (_, o) => _errorOutputLines.Add(ClearBOM(o)), OnStandardOutput = (_, o) => _standardOutputLines.Add(ClearBOM(o)), - WorkingDirectory = workingDirectory ?? Path.GetDirectoryName(command), + WorkingDirectory = workingDirectory, }; _process = ProcessFactory.Start(startInfo, cleanDefaultEnvironmentVariableIfCustomAreProvided); diff --git a/test/Utilities/Microsoft.Testing.TestInfrastructure/ProcessFactory.cs b/test/Utilities/Microsoft.Testing.TestInfrastructure/ProcessFactory.cs index da14874ff5..a7a7b0a8fa 100644 --- a/test/Utilities/Microsoft.Testing.TestInfrastructure/ProcessFactory.cs +++ b/test/Utilities/Microsoft.Testing.TestInfrastructure/ProcessFactory.cs @@ -9,7 +9,7 @@ public static IProcessHandle Start(ProcessConfiguration config, bool cleanDefaul { string fullPath = config.FileName; // Path.GetFullPath(startInfo.FileName); string workingDirectory = config.WorkingDirectory - .OrDefault(TempDirectory.TestSuiteDirectory); + .OrDefault(Path.GetDirectoryName(config.FileName).OrDefault(Directory.GetCurrentDirectory())); ProcessStartInfo processStartInfo = new() { diff --git a/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs b/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs index 88295869c5..f36305e3de 100644 --- a/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs +++ b/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs @@ -146,21 +146,7 @@ public string CopyFile(string filePath) [Obsolete("Don't use directly. Use TestSuiteDirectory property instead.")] private static string GetTestSuiteDirectory() - { - string testSuiteDirectory = System.IO.Path.Combine(RepoRoot, "artifacts", "tmp", Constants.BuildConfiguration, "testsuite"); - Directory.CreateDirectory(testSuiteDirectory); - - // Our tests were originally wrote before the enhanced dotnet test support for MTP. - // So, by default we use VSTest. - // We can start to gradually move some of the MTP tests to the new dotnet test experience. - // Note that we have tests that are actually VSTest-specific. - string dotnetConfig = System.IO.Path.Combine(testSuiteDirectory, "dotnet.config"); - File.WriteAllText(dotnetConfig, """ - [dotnet.test:runner] - name= "VSTest" - """); - return testSuiteDirectory; - } + => System.IO.Path.Combine(RepoRoot, "artifacts", "tmp", Constants.BuildConfiguration, "testsuite"); [Obsolete("Don't use directly. Use RepoRoot property instead.")] private static string GetRepoRoot() From adb848a3f802f8c6aa9e68fbcdb15c8a087d4063 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Sat, 8 Mar 2025 15:45:21 +0100 Subject: [PATCH 13/52] Adjust tests --- .../DotnetTestCliTests.cs | 2 +- .../MSBuildRunnerTests.cs | 12 ++++-------- .../MSTest.Acceptance.IntegrationTests/SdkTests.cs | 4 ++++ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/DotnetTestCliTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/DotnetTestCliTests.cs index 85a3cc1970..62e7e179be 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/DotnetTestCliTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/DotnetTestCliTests.cs @@ -25,7 +25,7 @@ public async Task DotnetTest_Should_Execute_Tests(string tfm, BuildConfiguration .PatchCodeWithReplace("$OutputType$", string.Empty) .PatchCodeWithReplace("$Extra$", string.Empty)); - DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"test -m:1 -nodeReuse:false {generator.TargetAssetPath}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path); + DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"test -m:1 -nodeReuse:false --directory {generator.TargetAssetPath}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path); // There is whitespace difference in output in parent and public repo that depends on the version of the dotnet SDK used. compilationResult.AssertOutputMatchesRegex(@"Passed!\s+-\s+Failed:\s+0,\s+Passed:\s+1,\s+Skipped:\s+0,\s+Total:\s+1"); diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSBuildRunnerTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSBuildRunnerTests.cs index 770b7eb021..22c8af88d3 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSBuildRunnerTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSBuildRunnerTests.cs @@ -39,13 +39,7 @@ public async Task MSBuildTestTarget_SingleAndMultiTfm_Should_Run_Solution_Tests( .PatchCodeWithReplace("$MicrosoftNETTestSdkVersion$", MicrosoftNETTestSdkVersion) .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$EnableMSTestRunner$", "true") - .PatchCodeWithReplace("$OutputType$", "Exe") - .PatchCodeWithReplace("$Extra$", command == DotnetTestVerb ? -""" - true - false -""" : - string.Empty)); + .PatchCodeWithReplace("$OutputType$", "Exe")); string projectContent = File.ReadAllText(Directory.GetFiles(generator.TargetAssetPath, "MSTestProject.csproj", SearchOption.AllDirectories).Single()); string testSourceContent = File.ReadAllText(Directory.GetFiles(generator.TargetAssetPath, "UnitTest1.cs", SearchOption.AllDirectories).Single()); @@ -66,7 +60,9 @@ public async Task MSBuildTestTarget_SingleAndMultiTfm_Should_Run_Solution_Tests( // Build the solution DotnetMuxerResult restoreResult = await DotnetCli.RunAsync($"restore -m:1 -nodeReuse:false {solution.SolutionFile} --configfile {nugetFile}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path); restoreResult.AssertOutputDoesNotContain("An approximate best match of"); - DotnetMuxerResult testResult = await DotnetCli.RunAsync($"{command} -m:1 -nodeReuse:false {solution.SolutionFile}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path); + + command = command == DotnetTestVerb ? $"{command} {solution.SolutionFile}" : $"{command} --solution {solution.SolutionFile}"; + DotnetMuxerResult testResult = await DotnetCli.RunAsync($"{command} -m:1 -nodeReuse:false", AcceptanceFixture.NuGetGlobalPackagesFolder.Path); if (isMultiTfm) { diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs index 21cf44541d..3dfdbd14e5 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs @@ -33,6 +33,10 @@ we end up with a -dev or -ci version which will lose resolution over -preview de +#file dotnet.config +[dotnet.test:runner] +name= "VSTest" + #file UnitTest1.cs using Microsoft.VisualStudio.TestTools.UnitTesting; From 95f1ae59e9ed8a534d298ad971e9e1f0caf2c3a2 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Sat, 8 Mar 2025 16:54:45 +0100 Subject: [PATCH 14/52] Fix test --- .../MSTest.Acceptance.IntegrationTests/MSBuildRunnerTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSBuildRunnerTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSBuildRunnerTests.cs index 22c8af88d3..392859408c 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSBuildRunnerTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSBuildRunnerTests.cs @@ -39,7 +39,8 @@ public async Task MSBuildTestTarget_SingleAndMultiTfm_Should_Run_Solution_Tests( .PatchCodeWithReplace("$MicrosoftNETTestSdkVersion$", MicrosoftNETTestSdkVersion) .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$EnableMSTestRunner$", "true") - .PatchCodeWithReplace("$OutputType$", "Exe")); + .PatchCodeWithReplace("$OutputType$", "Exe") + .PatchCodeWithReplace("$Extra$", string.Empty)); string projectContent = File.ReadAllText(Directory.GetFiles(generator.TargetAssetPath, "MSTestProject.csproj", SearchOption.AllDirectories).Single()); string testSourceContent = File.ReadAllText(Directory.GetFiles(generator.TargetAssetPath, "UnitTest1.cs", SearchOption.AllDirectories).Single()); From a2fdd3c02af51bcec14be5159aa3a4cf87d22f08 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Sat, 8 Mar 2025 17:27:56 +0100 Subject: [PATCH 15/52] Fix some tests --- .../PublishAotNonNativeTests.cs | 4 ++++ .../MSTest.Acceptance.IntegrationTests/SdkTests.cs | 4 ++++ .../MSBuildTests.Test.cs | 5 +++++ 3 files changed, 13 insertions(+) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/PublishAotNonNativeTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/PublishAotNonNativeTests.cs index 2ff4068225..35751ac26b 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/PublishAotNonNativeTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/PublishAotNonNativeTests.cs @@ -68,6 +68,10 @@ we end up with a -dev or -ci version which will lose resolution over -preview de +#file dotnet.config +[dotnet.test:runner] +name= "VSTest" + #file UnitTest1.cs using Microsoft.VisualStudio.TestTools.UnitTesting; diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs index 3dfdbd14e5..ef10064abc 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs @@ -465,6 +465,10 @@ public sealed class TestAssetFixture() : TestAssetFixtureBase(AcceptanceFixture. +#file dotnet.config +[dotnet.test:runner] +name= "VSTest" + #file UnitTest1.cs namespace AspireProject; diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Test.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Test.cs index 0953fbb993..1d8db95f43 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Test.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Test.cs @@ -158,6 +158,7 @@ await DotnetCli.RunAsync( [TestMethod] public async Task Invoke_DotnetTest_With_Incompatible_Arch() { + // TODO: Test with both old and new dotnet test experience. Architecture currentArchitecture = RuntimeInformation.ProcessArchitecture; string incompatibleArchitecture = currentArchitecture switch { @@ -314,6 +315,10 @@ public async Task TestingPlatformDisableCustomTestTarget_Should_Cause_UserDefine +#file dotnet.config +[dotnet.test:runner] +name= "VSTest" + #file Program.cs using Microsoft.Testing.Platform.Builder; using Microsoft.Testing.Platform.Capabilities.TestFramework; From bd1c4295b1ab0e68974d8650cc1a3417fe6cd6a1 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Sat, 8 Mar 2025 18:07:59 +0100 Subject: [PATCH 16/52] Adjust --- .../MSBuildRunnerTests.cs | 9 +++++++-- .../Helpers/AcceptanceTestBase.cs | 4 ++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSBuildRunnerTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSBuildRunnerTests.cs index 392859408c..4120f9ebfe 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSBuildRunnerTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSBuildRunnerTests.cs @@ -40,7 +40,12 @@ public async Task MSBuildTestTarget_SingleAndMultiTfm_Should_Run_Solution_Tests( .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) .PatchCodeWithReplace("$EnableMSTestRunner$", "true") .PatchCodeWithReplace("$OutputType$", "Exe") - .PatchCodeWithReplace("$Extra$", string.Empty)); + .PatchCodeWithReplace("$Extra$", command == DotnetTestVerb ? +""" + true + false +""" : + string.Empty)); string projectContent = File.ReadAllText(Directory.GetFiles(generator.TargetAssetPath, "MSTestProject.csproj", SearchOption.AllDirectories).Single()); string testSourceContent = File.ReadAllText(Directory.GetFiles(generator.TargetAssetPath, "UnitTest1.cs", SearchOption.AllDirectories).Single()); @@ -63,7 +68,7 @@ public async Task MSBuildTestTarget_SingleAndMultiTfm_Should_Run_Solution_Tests( restoreResult.AssertOutputDoesNotContain("An approximate best match of"); command = command == DotnetTestVerb ? $"{command} {solution.SolutionFile}" : $"{command} --solution {solution.SolutionFile}"; - DotnetMuxerResult testResult = await DotnetCli.RunAsync($"{command} -m:1 -nodeReuse:false", AcceptanceFixture.NuGetGlobalPackagesFolder.Path); + DotnetMuxerResult testResult = await DotnetCli.RunAsync($"{command} -m:1 -nodeReuse:false {solution.SolutionFile}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path); if (isMultiTfm) { diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceTestBase.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceTestBase.cs index 49429f20a8..b4703414ab 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceTestBase.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceTestBase.cs @@ -42,6 +42,10 @@ public void TestMethod1() { } } + +#file dotnet.config +[dotnet.test:runner] +name= "VSTest" """; static AcceptanceTestBase() From 82ca6d80ea777154e51b4c50c2f03b7a741583d7 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Sat, 8 Mar 2025 18:11:07 +0100 Subject: [PATCH 17/52] Adjust --- .../MSTest.Acceptance.IntegrationTests/DotnetTestCliTests.cs | 2 +- .../MSTest.Acceptance.IntegrationTests/MSBuildRunnerTests.cs | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/DotnetTestCliTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/DotnetTestCliTests.cs index 62e7e179be..85a3cc1970 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/DotnetTestCliTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/DotnetTestCliTests.cs @@ -25,7 +25,7 @@ public async Task DotnetTest_Should_Execute_Tests(string tfm, BuildConfiguration .PatchCodeWithReplace("$OutputType$", string.Empty) .PatchCodeWithReplace("$Extra$", string.Empty)); - DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"test -m:1 -nodeReuse:false --directory {generator.TargetAssetPath}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path); + DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"test -m:1 -nodeReuse:false {generator.TargetAssetPath}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path); // There is whitespace difference in output in parent and public repo that depends on the version of the dotnet SDK used. compilationResult.AssertOutputMatchesRegex(@"Passed!\s+-\s+Failed:\s+0,\s+Passed:\s+1,\s+Skipped:\s+0,\s+Total:\s+1"); diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSBuildRunnerTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSBuildRunnerTests.cs index 4120f9ebfe..770b7eb021 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSBuildRunnerTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSBuildRunnerTests.cs @@ -66,8 +66,6 @@ public async Task MSBuildTestTarget_SingleAndMultiTfm_Should_Run_Solution_Tests( // Build the solution DotnetMuxerResult restoreResult = await DotnetCli.RunAsync($"restore -m:1 -nodeReuse:false {solution.SolutionFile} --configfile {nugetFile}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path); restoreResult.AssertOutputDoesNotContain("An approximate best match of"); - - command = command == DotnetTestVerb ? $"{command} {solution.SolutionFile}" : $"{command} --solution {solution.SolutionFile}"; DotnetMuxerResult testResult = await DotnetCli.RunAsync($"{command} -m:1 -nodeReuse:false {solution.SolutionFile}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path); if (isMultiTfm) From 3b9a212d73ef04e995dba88887efa3ac3bd30640 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Sun, 9 Mar 2025 05:30:24 +0100 Subject: [PATCH 18/52] Fix tests --- .../DotnetTestCliTests.cs | 2 +- .../MSBuildRunnerTests.cs | 3 ++- .../PublishAotNonNativeTests.cs | 2 +- .../MSTest.Acceptance.IntegrationTests/SdkTests.cs | 8 +++++--- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/DotnetTestCliTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/DotnetTestCliTests.cs index 85a3cc1970..866bd1ed8f 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/DotnetTestCliTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/DotnetTestCliTests.cs @@ -25,7 +25,7 @@ public async Task DotnetTest_Should_Execute_Tests(string tfm, BuildConfiguration .PatchCodeWithReplace("$OutputType$", string.Empty) .PatchCodeWithReplace("$Extra$", string.Empty)); - DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"test -m:1 -nodeReuse:false {generator.TargetAssetPath}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path); + DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"test -m:1 -nodeReuse:false {generator.TargetAssetPath}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, workingDirectory: generator.TargetAssetPath); // There is whitespace difference in output in parent and public repo that depends on the version of the dotnet SDK used. compilationResult.AssertOutputMatchesRegex(@"Passed!\s+-\s+Failed:\s+0,\s+Passed:\s+1,\s+Skipped:\s+0,\s+Total:\s+1"); diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSBuildRunnerTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSBuildRunnerTests.cs index 770b7eb021..e826266a0b 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSBuildRunnerTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSBuildRunnerTests.cs @@ -66,7 +66,8 @@ public async Task MSBuildTestTarget_SingleAndMultiTfm_Should_Run_Solution_Tests( // Build the solution DotnetMuxerResult restoreResult = await DotnetCli.RunAsync($"restore -m:1 -nodeReuse:false {solution.SolutionFile} --configfile {nugetFile}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path); restoreResult.AssertOutputDoesNotContain("An approximate best match of"); - DotnetMuxerResult testResult = await DotnetCli.RunAsync($"{command} -m:1 -nodeReuse:false {solution.SolutionFile}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path); + + DotnetMuxerResult testResult = await DotnetCli.RunAsync($"{command} -m:1 -nodeReuse:false {solution.SolutionFile}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, workingDirectory: generator.TargetAssetPath); if (isMultiTfm) { diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/PublishAotNonNativeTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/PublishAotNonNativeTests.cs index 35751ac26b..fdcc62e704 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/PublishAotNonNativeTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/PublishAotNonNativeTests.cs @@ -24,7 +24,7 @@ public async Task RunTests_ThatEnablePublishAOT_ButDontBuildToNative() .PatchCodeWithReplace("$TargetFramework$", $"{TargetFrameworks.NetCurrent}") .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); - DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"test -c Debug {generator.TargetAssetPath}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, failIfReturnValueIsNotZero: false); + DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"test -c Debug {generator.TargetAssetPath}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, workingDirectory: generator.TargetAssetPath, failIfReturnValueIsNotZero: false); // In the real-world issue, access to path C:\Program Files\dotnet\ is denied, but we run this from a local .dotnet folder, where we have write access. // So instead of relying on the test run failing because of AccessDenied, we check the output, and see where TestResults were placed. diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs index ef10064abc..95789975f5 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs @@ -64,7 +64,7 @@ public async Task RunTests_With_VSTest(string multiTfm, BuildConfiguration build .PatchCodeWithReplace("$TargetFramework$", multiTfm) .PatchCodeWithReplace("$ExtraProperties$", "true")); - DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"test -c {buildConfiguration} {testAsset.TargetAssetPath}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path); + DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"test -c {buildConfiguration} {testAsset.TargetAssetPath}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, workingDirectory: testAsset.TargetAssetPath); Assert.AreEqual(0, compilationResult.ExitCode); compilationResult.AssertOutputMatchesRegex(@"Passed! - Failed: 0, Passed: 1, Skipped: 0, Total: 1, Duration: .* [m]?s - MSTestSdk.dll \(net9\.0\)"); @@ -91,7 +91,7 @@ public async Task RunTests_With_MSTestRunner_DotnetTest(string multiTfm, BuildCo .PatchCodeWithReplace("$TargetFramework$", multiTfm) .PatchCodeWithReplace("$ExtraProperties$", string.Empty)); - DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"test -c {buildConfiguration} {testAsset.TargetAssetPath}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path); + DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"test -c {buildConfiguration} {testAsset.TargetAssetPath}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, workingDirectory: testAsset.TargetAssetPath); Assert.AreEqual(0, compilationResult.ExitCode); compilationResult.AssertOutputMatchesRegex(@"Tests succeeded: .* \[net9\.0|x64\]"); @@ -359,6 +359,7 @@ public async Task EnablePlaywrightProperty_WhenUsingVSTest_AllowsToRunPlaywright DotnetMuxerResult dotnetTestResult = await DotnetCli.RunAsync( $"test {exeOrDllName}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, + workingDirectory: AssetFixture.PlaywrightProjectPath, failIfReturnValueIsNotZero: false, warnAsError: false, suppressPreviewDotNetMessage: false); @@ -402,6 +403,7 @@ public async Task EnableAspireProperty_WhenUsingVSTest_AllowsToRunAspireTests() DotnetMuxerResult dotnetTestResult = await DotnetCli.RunAsync( $"test {exeOrDllName}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, + workingDirectory: AssetFixture.AspireProjectPath, warnAsError: false, suppressPreviewDotNetMessage: false); Assert.AreEqual(0, dotnetTestResult.ExitCode); @@ -420,7 +422,7 @@ public async Task SettingIsTestApplicationToFalseReducesAddedExtensionsAndMakesP .PatchCodeWithReplace("$TargetFramework$", TargetFrameworks.NetCurrent) .PatchCodeWithReplace("$ExtraProperties$", "false")); - DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"test {testAsset.TargetAssetPath}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path); + DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"test {testAsset.TargetAssetPath}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, workingDirectory: testAsset.TargetAssetPath); Assert.AreEqual(0, compilationResult.ExitCode); From dbb9a9ef6c16a3c1fb8d9c83522677d777495f9f Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Sun, 9 Mar 2025 06:03:17 +0100 Subject: [PATCH 19/52] One more fix --- .../MSTest.Acceptance.IntegrationTests/SdkTests.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs index 95789975f5..a7f0590819 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs @@ -533,6 +533,10 @@ public async Task HomepageHasPlaywrightInTitleAndGetStartedLinkLinkingToTheIntro await Expect(Page).ToHaveURLAsync(new Regex(".*intro")); } } + +#file dotnet.config +[dotnet.test:runner] +name= "VSTest" """; public string AspireProjectPath => GetAssetPath(AspireProjectName); From 68f64cbfa530d195ab02bfba7ad335ae08b99d03 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Sun, 9 Mar 2025 06:38:47 +0100 Subject: [PATCH 20/52] More progress --- .../MSBuildTests.Solution.cs | 2 +- .../MSBuildTests.Test.cs | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Solution.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Solution.cs index c9c13b4a22..58aefa8ceb 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Solution.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Solution.cs @@ -55,7 +55,7 @@ public async Task MSBuildTests_UseMSBuildTestInfrastructure_Should_Run_Solution_ // Build the solution DotnetMuxerResult restoreResult = await DotnetCli.RunAsync($"restore -nodeReuse:false {solution.SolutionFile} --configfile {nugetFile}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path); restoreResult.AssertOutputDoesNotContain("An approximate best match of"); - DotnetMuxerResult testResult = await DotnetCli.RunAsync($"{command} -nodeReuse:false {solution.SolutionFile}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path); + DotnetMuxerResult testResult = await DotnetCli.RunAsync($"{command} -nodeReuse:false {solution.SolutionFile}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, workingDirectory: generator.TargetAssetPath); if (isMultiTfm) { diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Test.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Test.cs index 5b1e2f5dc3..546d0ec2c0 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Test.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Test.cs @@ -86,10 +86,12 @@ private async Task InvokeTestingPlatform_Target_Should_Build_Without_Warnings_An DotnetMuxerResult compilationResult = testCommand.StartsWith("test", StringComparison.OrdinalIgnoreCase) ? await DotnetCli.RunAsync( $"{testCommand} -p:Configuration={compilationMode} -p:nodeReuse=false \"{testAsset.TargetAssetPath}\" -- --treenode-filter --results-directory \"{testResultFolder}\"", - AcceptanceFixture.NuGetGlobalPackagesFolder.Path) + AcceptanceFixture.NuGetGlobalPackagesFolder.Path, + workingDirectory: testAsset.TargetAssetPath) : await DotnetCli.RunAsync( $"{testCommand} -p:TestingPlatformCommandLineArguments=\"--treenode-filter --results-directory \"{testResultFolder}\"\" -p:Configuration={compilationMode} -p:nodeReuse=false \"{testAsset.TargetAssetPath}\"", - AcceptanceFixture.NuGetGlobalPackagesFolder.Path); + AcceptanceFixture.NuGetGlobalPackagesFolder.Path, + workingDirectory: testAsset.TargetAssetPath); foreach (string tfmToAssert in tfmsToAssert) { @@ -178,6 +180,7 @@ public async Task Invoke_DotnetTest_With_Incompatible_Arch() DotnetMuxerResult result = await DotnetCli.RunAsync( $"test --arch {incompatibleArchitecture} -p:TestingPlatformDotnetTestSupport=True \"{testAsset.TargetAssetPath}\"", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, + workingDirectory: testAsset.TargetAssetPath, failIfReturnValueIsNotZero: false); // On Windows, we run the exe directly. From 569c16d16f9970146fa297cb347c8eb87ba181f9 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Sun, 9 Mar 2025 07:25:16 +0100 Subject: [PATCH 21/52] Fix more tests --- .../MSBuildTests.Solution.cs | 6 +++++- .../MSBuildTests.Test.cs | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Solution.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Solution.cs index 58aefa8ceb..1a9051ab1a 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Solution.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Solution.cs @@ -42,6 +42,10 @@ public async Task MSBuildTests_UseMSBuildTestInfrastructure_Should_Run_Solution_ string solutionFolder = Path.Combine(tempDirectory.Path, "Solution"); VSSolution solution = new(solutionFolder, "MSTestSolution"); string nugetFile = solution.AddOrUpdateFileContent("Nuget.config", nugetConfigContent); + solution.AddOrUpdateFileContent("dotnet.config", """ + [dotnet.test:runner] + name= "VSTest" + """); for (int i = 0; i < 3; i++) { CSharpProject project = solution.CreateCSharpProject($"TestProject{i}", isMultiTfm ? singleTfmOrMultiTfm.Split(';') : [singleTfmOrMultiTfm]); @@ -55,7 +59,7 @@ public async Task MSBuildTests_UseMSBuildTestInfrastructure_Should_Run_Solution_ // Build the solution DotnetMuxerResult restoreResult = await DotnetCli.RunAsync($"restore -nodeReuse:false {solution.SolutionFile} --configfile {nugetFile}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path); restoreResult.AssertOutputDoesNotContain("An approximate best match of"); - DotnetMuxerResult testResult = await DotnetCli.RunAsync($"{command} -nodeReuse:false {solution.SolutionFile}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, workingDirectory: generator.TargetAssetPath); + DotnetMuxerResult testResult = await DotnetCli.RunAsync($"{command} -nodeReuse:false {solution.SolutionFile}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, workingDirectory: solution.SolutionFile); if (isMultiTfm) { diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Test.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Test.cs index 546d0ec2c0..4395e9601f 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Test.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Test.cs @@ -54,7 +54,7 @@ private async Task InvokeTestingPlatform_Target_Should_Execute_Tests_Without_Sho .PatchCodeWithReplace("$AssertValue$", testSucceeded.ToString().ToLowerInvariant()) .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion)); string testResultFolder = Path.Combine(testAsset.TargetAssetPath, Guid.NewGuid().ToString("N")); - DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"{testCommand} -p:TestingPlatformCommandLineArguments=\"--results-directory %22{testResultFolder}%22\" -p:Configuration={compilationMode} -p:nodeReuse=false \"{testAsset.TargetAssetPath}\"", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, failIfReturnValueIsNotZero: false); + DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"{testCommand} -p:TestingPlatformCommandLineArguments=\"--results-directory %22{testResultFolder}%22\" -p:Configuration={compilationMode} -p:nodeReuse=false \"{testAsset.TargetAssetPath}\"", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, workingDirectory: testAsset.TargetAssetPath, failIfReturnValueIsNotZero: false); foreach (string tfmToAssert in tfmsToAssert) { From ebc54ff61af2b14ac97584b9a1970a5993e0da4d Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Sun, 9 Mar 2025 08:19:03 +0100 Subject: [PATCH 22/52] More test fixes --- .../MSBuildTests.Solution.cs | 2 +- .../MSBuildTests.Test.cs | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Solution.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Solution.cs index 1a9051ab1a..87a3c84887 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Solution.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Solution.cs @@ -59,7 +59,7 @@ public async Task MSBuildTests_UseMSBuildTestInfrastructure_Should_Run_Solution_ // Build the solution DotnetMuxerResult restoreResult = await DotnetCli.RunAsync($"restore -nodeReuse:false {solution.SolutionFile} --configfile {nugetFile}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path); restoreResult.AssertOutputDoesNotContain("An approximate best match of"); - DotnetMuxerResult testResult = await DotnetCli.RunAsync($"{command} -nodeReuse:false {solution.SolutionFile}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, workingDirectory: solution.SolutionFile); + DotnetMuxerResult testResult = await DotnetCli.RunAsync($"{command} -nodeReuse:false {solution.SolutionFile}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, workingDirectory: solution.FolderPath); if (isMultiTfm) { diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Test.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Test.cs index 4395e9601f..1ed1fe2e16 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Test.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Test.cs @@ -147,6 +147,7 @@ public async Task Invoke_DotnetTest_With_Arch_Switch_x86_Should_Work() await DotnetCli.RunAsync( $"test --arch x86 -p:TestingPlatformDotnetTestSupport=True -p:Configuration=Release -p:nodeReuse=false \"{testAsset.TargetAssetPath}\"", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, + workingDirectory: testAsset.TargetAssetPath, environmentVariables: dotnetRootX86, failIfReturnValueIsNotZero: false); @@ -235,6 +236,7 @@ public async Task Invoke_DotnetTest_With_DOTNET_HOST_PATH_Should_Work() await DotnetCli.RunAsync( $"test -p:TestingPlatformDotnetTestSupport=True -p:Configuration=Release -p:nodeReuse=false \"{testAsset.TargetAssetPath}\"", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, + workingDirectory: testAsset.TargetAssetPath, environmentVariables: dotnetHostPathEnvVar, failIfReturnValueIsNotZero: false); @@ -282,7 +284,7 @@ public async Task InvokeTestingPlatform_Target_Showing_Error_And_Do_Not_Capture_ .PatchCodeWithReplace("$TargetFrameworks$", $"{tfm}") .PatchCodeWithReplace("$AssertValue$", testSucceeded.ToString().ToLowerInvariant()) .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion)); - DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"{testCommand} -p:TestingPlatformShowTestsFailure=True -p:TestingPlatformCaptureOutput=False -p:Configuration={compilationMode} -p:nodeReuse=false {testAsset.TargetAssetPath}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, failIfReturnValueIsNotZero: false); + DotnetMuxerResult compilationResult = await DotnetCli.RunAsync($"{testCommand} -p:TestingPlatformShowTestsFailure=True -p:TestingPlatformCaptureOutput=False -p:Configuration={compilationMode} -p:nodeReuse=false {testAsset.TargetAssetPath}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, workingDirectory: testAsset.TargetAssetPath, failIfReturnValueIsNotZero: false); compilationResult.AssertOutputContains("error test failed: Test2 ("); compilationResult.AssertOutputContains("FAILED: Expected 'true', but got 'false'."); From 2b9e3a38dd4e05bdf1204b36b2d4c8bf2a89a36a Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Sun, 9 Mar 2025 09:13:50 +0100 Subject: [PATCH 23/52] Use dotnet test in CI --- azure-pipelines.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7ab31f8fe6..1e4c912481 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -94,12 +94,7 @@ stages: displayName: Build - ${{ if eq(parameters.SkipTests, False) }}: - # -ci is allowing to import some environment variables and some required configurations - # -nobl avoids overwriting build binlog with binlog from tests - - script: Test.cmd - -configuration $(_BuildConfig) - -ci - -nobl + - script: dotnet test -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog name: Test displayName: Test From a6b7c83f880286ae60d81cccef6bd26c8f870399 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Sun, 9 Mar 2025 09:14:26 +0100 Subject: [PATCH 24/52] Fix --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1e4c912481..7a741ad9ca 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -94,7 +94,7 @@ stages: displayName: Build - ${{ if eq(parameters.SkipTests, False) }}: - - script: dotnet test -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog + - script: $(Build.SourcesDirectory)/.dotnet/dotnet test -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog name: Test displayName: Test From 1c5748ccecc7dd681a2c0d9ed3366aa59082014f Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Sun, 9 Mar 2025 09:34:06 +0100 Subject: [PATCH 25/52] Fix for Playground not considered MTP --- samples/Playground/Playground.csproj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/samples/Playground/Playground.csproj b/samples/Playground/Playground.csproj index a4123f3a05..b8570b8348 100644 --- a/samples/Playground/Playground.csproj +++ b/samples/Playground/Playground.csproj @@ -6,6 +6,8 @@ enable false $(NoWarn);NETSDK1023 + + true From d0184006cd7fb35135edccc0157300462375562d Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Sun, 9 Mar 2025 10:20:12 +0100 Subject: [PATCH 26/52] Fix --- azure-pipelines.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7a741ad9ca..f36ababe7c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -94,7 +94,10 @@ stages: displayName: Build - ${{ if eq(parameters.SkipTests, False) }}: - - script: $(Build.SourcesDirectory)/.dotnet/dotnet test -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog + # Because the build step is using -ci flag, restore is done in a local .packages directory. + # We need to pass NUGET_PACKAGES so that when dotnet test is doing evaluation phase on the projects, it can resolve .props/.targets from packages and import them. + # Otherwise, props/targets are not imported. It's important that they are imported so that IsTestingPlatformApplication ends up being set. + - script: $(Build.SourcesDirectory)/.dotnet/dotnet test -p:NUGET_PACKAGES=$(BUILD.SOURCESDIRECTORY)\.packages -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog name: Test displayName: Test From c24ee022c20dd9d33ea772d055ecc1a2388b52ab Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Sun, 9 Mar 2025 11:19:07 +0100 Subject: [PATCH 27/52] Fix for MSBuild unit tests --- .../Microsoft.Testing.Platform.MSBuild.UnitTests.csproj | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests.csproj b/test/UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests.csproj index dcff0824b4..59e0642311 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests.csproj +++ b/test/UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests.csproj @@ -4,6 +4,10 @@ net8.0;net9.0 true Exe + + + true + From 85e9494c69dac076f450ac7ee2bdd1659d9c6432 Mon Sep 17 00:00:00 2001 From: Youssef Victor Date: Mon, 10 Mar 2025 13:24:38 +0100 Subject: [PATCH 28/52] Update dotnet.config --- dotnet.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet.config b/dotnet.config index 63ba841e87..1271fc9236 100644 --- a/dotnet.config +++ b/dotnet.config @@ -1,2 +1,2 @@ [dotnet.test:runner] -name= "MicrosoftTestingPlatform" +name= "Microsoft.Testing.Platform" From 7c71858ff213a1fa866d086bc2fa8f6d0cc0611a Mon Sep 17 00:00:00 2001 From: Youssef Victor Date: Mon, 10 Mar 2025 13:25:11 +0100 Subject: [PATCH 29/52] Update SDK --- global.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/global.json b/global.json index 9f471afd76..2d8d759c6d 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "tools": { - "dotnet": "10.0.100-preview.3.25125.5", + "dotnet": "10.0.100-preview.3.25160.9", "runtimes": { "dotnet": [ "3.1.32", @@ -23,7 +23,7 @@ } }, "sdk": { - "version": "10.0.100-preview.3.25125.5", + "version": "10.0.100-preview.3.25160.9", "allowPrerelease": true, "rollForward": "latestFeature" }, From aba6450e9a3d41c6e6a8db248d247db022ef0a90 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 10 Mar 2025 15:04:17 +0100 Subject: [PATCH 30/52] Trace --- azure-pipelines.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f36ababe7c..a0558a0731 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -98,6 +98,8 @@ stages: # We need to pass NUGET_PACKAGES so that when dotnet test is doing evaluation phase on the projects, it can resolve .props/.targets from packages and import them. # Otherwise, props/targets are not imported. It's important that they are imported so that IsTestingPlatformApplication ends up being set. - script: $(Build.SourcesDirectory)/.dotnet/dotnet test -p:NUGET_PACKAGES=$(BUILD.SOURCESDIRECTORY)\.packages -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog + env: + DOTNET_CLI_TEST_TRACEFILE: $(Build.SourcesDirectory)/artifacts/TestResults/dotnet-test-trace-logs.txt name: Test displayName: Test From 495afb33630268dd396e5a0c1418541b34b24064 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 10 Mar 2025 15:25:23 +0100 Subject: [PATCH 31/52] Fix --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a0558a0731..7b0194e8f4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -99,7 +99,7 @@ stages: # Otherwise, props/targets are not imported. It's important that they are imported so that IsTestingPlatformApplication ends up being set. - script: $(Build.SourcesDirectory)/.dotnet/dotnet test -p:NUGET_PACKAGES=$(BUILD.SOURCESDIRECTORY)\.packages -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog env: - DOTNET_CLI_TEST_TRACEFILE: $(Build.SourcesDirectory)/artifacts/TestResults/dotnet-test-trace-logs.txt + DOTNET_CLI_TEST_TRACEFILE: $(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/dotnet-test-trace-logs.txt name: Test displayName: Test From 91ef8d0fd2b6da996cebdfab3bd425e5ed30daa3 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 10 Mar 2025 15:31:19 +0100 Subject: [PATCH 32/52] Create directory --- azure-pipelines.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7b0194e8f4..723633f2d5 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -94,6 +94,10 @@ stages: displayName: Build - ${{ if eq(parameters.SkipTests, False) }}: + + # Workaround for https://github.com/microsoft/testfx/issues/5205 + - powershell: New-Item -Path "$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)" -Type Directory + # Because the build step is using -ci flag, restore is done in a local .packages directory. # We need to pass NUGET_PACKAGES so that when dotnet test is doing evaluation phase on the projects, it can resolve .props/.targets from packages and import them. # Otherwise, props/targets are not imported. It's important that they are imported so that IsTestingPlatformApplication ends up being set. From abf88e9640128a4ad8ae633f15730fdcb34c4340 Mon Sep 17 00:00:00 2001 From: Youssef Victor Date: Tue, 11 Mar 2025 15:26:58 +0100 Subject: [PATCH 33/52] Update SDK --- global.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/global.json b/global.json index 2d8d759c6d..15e9779ac6 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "tools": { - "dotnet": "10.0.100-preview.3.25160.9", + "dotnet": "10.0.100-preview.3.25161.5", "runtimes": { "dotnet": [ "3.1.32", @@ -23,7 +23,7 @@ } }, "sdk": { - "version": "10.0.100-preview.3.25160.9", + "version": "10.0.100-preview.3.25161.5", "allowPrerelease": true, "rollForward": "latestFeature" }, From d70e862caf3d78f53483432ddcec73bf94178cf2 Mon Sep 17 00:00:00 2001 From: Youssef Victor Date: Wed, 12 Mar 2025 12:37:09 +0100 Subject: [PATCH 34/52] Update SDK again --- global.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/global.json b/global.json index 15e9779ac6..a768ffc87a 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "tools": { - "dotnet": "10.0.100-preview.3.25161.5", + "dotnet": "10.0.100-preview.3.25162.14", "runtimes": { "dotnet": [ "3.1.32", @@ -23,7 +23,7 @@ } }, "sdk": { - "version": "10.0.100-preview.3.25161.5", + "version": "10.0.100-preview.3.25162.14", "allowPrerelease": true, "rollForward": "latestFeature" }, From 2cf00d1d9d8eeaa3a4a0f560b2062dea236b96a9 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 17 Mar 2025 09:31:29 +0100 Subject: [PATCH 35/52] Update --- global.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/global.json b/global.json index 5c3781c14d..bcde79d857 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "tools": { - "dotnet": "10.0.100-preview.3.25162.14", + "dotnet": "10.0.100-preview.3.25166.18", "runtimes": { "dotnet": [ "3.1.32", @@ -23,7 +23,7 @@ } }, "sdk": { - "version": "10.0.100-preview.3.25162.14", + "version": "10.0.100-preview.3.25166.18", "allowPrerelease": true, "rollForward": "latestFeature" }, From 4027275599becfe710e630e5bf2c05f80971baaf Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 17 Mar 2025 09:45:53 +0100 Subject: [PATCH 36/52] Adjust arguments --- azure-pipelines.yml | 24 ++++++++++-------------- test/Directory.Build.targets | 11 +++++++++++ 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 723633f2d5..b4049907e3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -102,8 +102,6 @@ stages: # We need to pass NUGET_PACKAGES so that when dotnet test is doing evaluation phase on the projects, it can resolve .props/.targets from packages and import them. # Otherwise, props/targets are not imported. It's important that they are imported so that IsTestingPlatformApplication ends up being set. - script: $(Build.SourcesDirectory)/.dotnet/dotnet test -p:NUGET_PACKAGES=$(BUILD.SOURCESDIRECTORY)\.packages -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog - env: - DOTNET_CLI_TEST_TRACEFILE: $(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/dotnet-test-trace-logs.txt name: Test displayName: Test @@ -161,13 +159,12 @@ stages: displayName: Build - ${{ if eq(parameters.SkipTests, False) }}: - # --ci is allowing to import some environment variables and some required configurations - # --nobl avoids overwriting build binlog with binlog from tests - - script: | - chmod +x ./test.sh - ./test.sh --configuration $(_BuildConfig) --ci --test --integrationTest --nobl + # Because the build step is using -ci flag, restore is done in a local .packages directory. + # We need to pass NUGET_PACKAGES so that when dotnet test is doing evaluation phase on the projects, it can resolve .props/.targets from packages and import them. + # Otherwise, props/targets are not imported. It's important that they are imported so that IsTestingPlatformApplication ends up being set. + - script: $(Build.SourcesDirectory)/.dotnet/dotnet test -p:NUGET_PACKAGES=$(BUILD.SOURCESDIRECTORY)\.packages -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog name: Test - displayName: Tests + displayName: Test # This step is only helpful for diagnosing some issues with vstest/test host that would not appear # through the console or trx @@ -216,13 +213,12 @@ stages: displayName: Build - ${{ if eq(parameters.SkipTests, False) }}: - # --ci is allowing to import some environment variables and some required configurations - # --nobl avoids overwriting build binlog with binlog from tests - - script: | - chmod +x ./test.sh - ./test.sh --configuration $(_BuildConfig) --ci --test --integrationTest --nobl + # Because the build step is using -ci flag, restore is done in a local .packages directory. + # We need to pass NUGET_PACKAGES so that when dotnet test is doing evaluation phase on the projects, it can resolve .props/.targets from packages and import them. + # Otherwise, props/targets are not imported. It's important that they are imported so that IsTestingPlatformApplication ends up being set. + - script: $(Build.SourcesDirectory)/.dotnet/dotnet test -p:NUGET_PACKAGES=$(BUILD.SOURCESDIRECTORY)\.packages -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog name: Test - displayName: Tests + displayName: Test # This step is only helpful for diagnosing some issues with vstest/test host that would not appear # through the console or trx diff --git a/test/Directory.Build.targets b/test/Directory.Build.targets index d74f48bbda..b695648f6a 100644 --- a/test/Directory.Build.targets +++ b/test/Directory.Build.targets @@ -17,6 +17,17 @@ $(TestRunnerAdditionalArguments) --hangdump --hangdump-timeout 15m $(TestRunnerAdditionalArguments) --coverage --coverage-settings $(RepoRoot)test/coverage.config --coverage-output $(ModuleName).coverage + + + <_TestArchitecture>$(PlatformTarget) + <_TestArchitecture Condition="'$(PlatformTarget)' == '' or '$(PlatformTarget)' == 'AnyCpu'">x64 + + <_ResultFileNameNoExt>$(MSBuildProjectName)_$(TargetFramework)_$(_TestArchitecture) + $(ArtifactsTestResultsDir)$(_ResultFileNameNoExt).trx + <_TestResultTrxFileName>$([System.IO.Path]::GetFileName('$(ResultsTrxPath)')) + <_TestResultDirectory>$([System.IO.Path]::GetDirectoryName('$(ResultsTrxPath)')) + $(TestingPlatformCommandLineArguments) --report-trx --report-trx-filename "$(_TestResultTrxFileName)" --results-directory "$(_TestResultDirectory)" $(TestRunnerAdditionalArguments) + From 970a300de579c1c917f9fea268509b95ad799b29 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 17 Mar 2025 11:11:01 +0100 Subject: [PATCH 37/52] Import targets --- test/Directory.Build.targets | 1 + 1 file changed, 1 insertion(+) diff --git a/test/Directory.Build.targets b/test/Directory.Build.targets index b695648f6a..4c84c3db2d 100644 --- a/test/Directory.Build.targets +++ b/test/Directory.Build.targets @@ -43,6 +43,7 @@ + From 9eb44882553003b76f956d24acdf1391c1a6ad27 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 17 Mar 2025 11:37:56 +0100 Subject: [PATCH 38/52] Build all tests --- eng/Build.props | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/eng/Build.props b/eng/Build.props index 57aac3a459..b8e10fd349 100644 --- a/eng/Build.props +++ b/eng/Build.props @@ -12,10 +12,7 @@ - - - - + From d97b4092a80df88e6e2dd20b17942d4893682fb2 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 17 Mar 2025 12:01:39 +0100 Subject: [PATCH 39/52] Adjust for non-Windows --- .../MSTestAdapter.PlatformServices.UnitTests.csproj | 3 ++- .../MSTestAdapter.UnitTests/MSTestAdapter.UnitTests.csproj | 3 ++- .../TestFramework.UnitTests/TestFramework.UnitTests.csproj | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/MSTestAdapter.PlatformServices.UnitTests.csproj b/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/MSTestAdapter.PlatformServices.UnitTests.csproj index c3998ee42c..de6630035c 100644 --- a/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/MSTestAdapter.PlatformServices.UnitTests.csproj +++ b/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/MSTestAdapter.PlatformServices.UnitTests.csproj @@ -2,7 +2,8 @@ net48 - net462;$(NetStandardNetFrameworkHolder);netcoreapp3.1;net6.0;$(WinUiMinimum) + net462;$(NetStandardNetFrameworkHolder);netcoreapp3.1;net6.0 + $(TargetFrameworks);WinUiMinimum true true true diff --git a/test/UnitTests/MSTestAdapter.UnitTests/MSTestAdapter.UnitTests.csproj b/test/UnitTests/MSTestAdapter.UnitTests/MSTestAdapter.UnitTests.csproj index 5814b209bb..524a4dad38 100644 --- a/test/UnitTests/MSTestAdapter.UnitTests/MSTestAdapter.UnitTests.csproj +++ b/test/UnitTests/MSTestAdapter.UnitTests/MSTestAdapter.UnitTests.csproj @@ -2,7 +2,8 @@ net48 - net6.0;net462;$(NetStandardNetFrameworkHolder);netcoreapp3.1;$(WinUiMinimum) + net6.0;net462;$(NetStandardNetFrameworkHolder);netcoreapp3.1 + $(TargetFrameworks);WinUiMinimum true Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests diff --git a/test/UnitTests/TestFramework.UnitTests/TestFramework.UnitTests.csproj b/test/UnitTests/TestFramework.UnitTests/TestFramework.UnitTests.csproj index 1ba5682a20..350bdb7213 100644 --- a/test/UnitTests/TestFramework.UnitTests/TestFramework.UnitTests.csproj +++ b/test/UnitTests/TestFramework.UnitTests/TestFramework.UnitTests.csproj @@ -2,7 +2,8 @@ net48 - net6.0;net462;$(NetStandardNetFrameworkHolder);netcoreapp3.1;$(WinUiMinimum) + net6.0;net462;$(NetStandardNetFrameworkHolder);netcoreapp3.1 + $(TargetFrameworks);WinUiMinimum true Microsoft.VisualStudio.TestPlatform.TestFramework.UnitTests Microsoft.VisualStudio.TestPlatform.TestFramework.UnitTests From 68cd58f4ed05f6ef9d03d128dc570ee62b67de1a Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Mon, 17 Mar 2025 12:33:36 +0100 Subject: [PATCH 40/52] Fix --- .../MSTestAdapter.PlatformServices.UnitTests.csproj | 2 +- .../MSTestAdapter.UnitTests/MSTestAdapter.UnitTests.csproj | 2 +- .../TestFramework.UnitTests/TestFramework.UnitTests.csproj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/MSTestAdapter.PlatformServices.UnitTests.csproj b/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/MSTestAdapter.PlatformServices.UnitTests.csproj index de6630035c..46cd16bf11 100644 --- a/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/MSTestAdapter.PlatformServices.UnitTests.csproj +++ b/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/MSTestAdapter.PlatformServices.UnitTests.csproj @@ -3,7 +3,7 @@ net48 net462;$(NetStandardNetFrameworkHolder);netcoreapp3.1;net6.0 - $(TargetFrameworks);WinUiMinimum + $(TargetFrameworks);$(WinUiMinimum) true true true diff --git a/test/UnitTests/MSTestAdapter.UnitTests/MSTestAdapter.UnitTests.csproj b/test/UnitTests/MSTestAdapter.UnitTests/MSTestAdapter.UnitTests.csproj index 524a4dad38..2f3eddd243 100644 --- a/test/UnitTests/MSTestAdapter.UnitTests/MSTestAdapter.UnitTests.csproj +++ b/test/UnitTests/MSTestAdapter.UnitTests/MSTestAdapter.UnitTests.csproj @@ -3,7 +3,7 @@ net48 net6.0;net462;$(NetStandardNetFrameworkHolder);netcoreapp3.1 - $(TargetFrameworks);WinUiMinimum + $(TargetFrameworks);$(WinUiMinimum) true Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests diff --git a/test/UnitTests/TestFramework.UnitTests/TestFramework.UnitTests.csproj b/test/UnitTests/TestFramework.UnitTests/TestFramework.UnitTests.csproj index 350bdb7213..2274f24540 100644 --- a/test/UnitTests/TestFramework.UnitTests/TestFramework.UnitTests.csproj +++ b/test/UnitTests/TestFramework.UnitTests/TestFramework.UnitTests.csproj @@ -3,7 +3,7 @@ net48 net6.0;net462;$(NetStandardNetFrameworkHolder);netcoreapp3.1 - $(TargetFrameworks);WinUiMinimum + $(TargetFrameworks);$(WinUiMinimum) true Microsoft.VisualStudio.TestPlatform.TestFramework.UnitTests Microsoft.VisualStudio.TestPlatform.TestFramework.UnitTests From 7f5df521737ca6f4c01429b1de3fbb9c02efbca1 Mon Sep 17 00:00:00 2001 From: Youssef Victor Date: Mon, 17 Mar 2025 13:23:11 +0100 Subject: [PATCH 41/52] Specify DOTNET_ROOT --- azure-pipelines.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b4049907e3..1160f5e3be 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -104,6 +104,9 @@ stages: - script: $(Build.SourcesDirectory)/.dotnet/dotnet test -p:NUGET_PACKAGES=$(BUILD.SOURCESDIRECTORY)\.packages -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog name: Test displayName: Test + env: + DOTNET_ROOT: $(Build.SourcesDirectory)/.dotnet + # This step is only helpful for diagnosing some issues with vstest/test host that would not appear # through the console or trx From 3d0ccc98969543f0a4541c416f4621fdd032fecf Mon Sep 17 00:00:00 2001 From: Youssef Victor Date: Mon, 17 Mar 2025 13:48:23 +0100 Subject: [PATCH 42/52] Use env variable --- azure-pipelines.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1160f5e3be..40434de131 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -95,17 +95,15 @@ stages: - ${{ if eq(parameters.SkipTests, False) }}: - # Workaround for https://github.com/microsoft/testfx/issues/5205 - - powershell: New-Item -Path "$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)" -Type Directory - # Because the build step is using -ci flag, restore is done in a local .packages directory. # We need to pass NUGET_PACKAGES so that when dotnet test is doing evaluation phase on the projects, it can resolve .props/.targets from packages and import them. # Otherwise, props/targets are not imported. It's important that they are imported so that IsTestingPlatformApplication ends up being set. - - script: $(Build.SourcesDirectory)/.dotnet/dotnet test -p:NUGET_PACKAGES=$(BUILD.SOURCESDIRECTORY)\.packages -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog + - script: $(Build.SourcesDirectory)/.dotnet/dotnet test -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog name: Test displayName: Test env: DOTNET_ROOT: $(Build.SourcesDirectory)/.dotnet + NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages # This step is only helpful for diagnosing some issues with vstest/test host that would not appear @@ -165,9 +163,12 @@ stages: # Because the build step is using -ci flag, restore is done in a local .packages directory. # We need to pass NUGET_PACKAGES so that when dotnet test is doing evaluation phase on the projects, it can resolve .props/.targets from packages and import them. # Otherwise, props/targets are not imported. It's important that they are imported so that IsTestingPlatformApplication ends up being set. - - script: $(Build.SourcesDirectory)/.dotnet/dotnet test -p:NUGET_PACKAGES=$(BUILD.SOURCESDIRECTORY)\.packages -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog + - script: $(Build.SourcesDirectory)/.dotnet/dotnet test -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog name: Test displayName: Test + env: + DOTNET_ROOT: $(Build.SourcesDirectory)/.dotnet + NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages # This step is only helpful for diagnosing some issues with vstest/test host that would not appear # through the console or trx @@ -219,9 +220,12 @@ stages: # Because the build step is using -ci flag, restore is done in a local .packages directory. # We need to pass NUGET_PACKAGES so that when dotnet test is doing evaluation phase on the projects, it can resolve .props/.targets from packages and import them. # Otherwise, props/targets are not imported. It's important that they are imported so that IsTestingPlatformApplication ends up being set. - - script: $(Build.SourcesDirectory)/.dotnet/dotnet test -p:NUGET_PACKAGES=$(BUILD.SOURCESDIRECTORY)\.packages -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog + - script: $(Build.SourcesDirectory)/.dotnet/dotnet test -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog name: Test displayName: Test + env: + DOTNET_ROOT: $(Build.SourcesDirectory)/.dotnet + NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages # This step is only helpful for diagnosing some issues with vstest/test host that would not appear # through the console or trx From 28015199e5636d9615f0472e6620233a8ee1892b Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Tue, 18 Mar 2025 11:33:19 +0100 Subject: [PATCH 43/52] Is Windows green now? --- .../RunsettingsTests.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/RunsettingsTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/RunsettingsTests.cs index b040a3170b..9944f0d74f 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/RunsettingsTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/RunsettingsTests.cs @@ -71,8 +71,12 @@ public async Task UnsupportedRunSettingsEntriesAreFlagged_Localization(string? t switch (expectedLocale) { case "fr-FR": - testHostResult.AssertOutputContains("Les loggers Runsettings ne sont pas pris en charge par Microsoft.Testing.Platform et seront ignorés"); - testHostResult.AssertOutputContains("Les datacollecteurs Runsettings ne sont pas pris en charge par Microsoft.Testing.Platform et seront ignorés"); + // Using regex for the "é" of ignorés as something with encoding doesn't work properly. + // The é shows correctly when invoking with Arcade, but not with dotnet test. + // This is probably because Arcade infra uses Exec MSBuild task, which seems to be having extra logic around handling encoding. + // See https://github.com/dotnet/msbuild/blob/bcc2dc6a6509ffb63f1253a9bbbaaa233bd53a50/src/Tasks/Exec.cs + testHostResult.AssertOutputMatchesRegex(@"Les loggers Runsettings ne sont pas pris en charge par Microsoft\.Testing\.Platform et seront ignor.*?s"); + testHostResult.AssertOutputMatchesRegex(@"Les datacollecteurs Runsettings ne sont pas pris en charge par Microsoft\.Testing\.Platform et seront ignor.*?s"); break; case "it-IT": testHostResult.AssertOutputContains("I logger Runsettings non sono supportati da Microsoft.Testing.Platform e verranno ignorati"); From ac2f2eca0784a9bc157d956a84aee9aeaa25872d Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Tue, 18 Mar 2025 12:10:54 +0100 Subject: [PATCH 44/52] Use slnf --- NonWindowsTests.slnf | 14 ++++++++++++++ eng/Build.props | 3 +-- 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 NonWindowsTests.slnf diff --git a/NonWindowsTests.slnf b/NonWindowsTests.slnf new file mode 100644 index 0000000000..c43f27f4bf --- /dev/null +++ b/NonWindowsTests.slnf @@ -0,0 +1,14 @@ +{ + "solution": { + "path": "TestFx.sln", + "projects": [ + "test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests.csproj", + "test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSTest.Acceptance.IntegrationTests.csproj", + "test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Microsoft.Testing.Extensions.UnitTests.csproj", + "test/UnitTests/Microsoft.Testing.Extensions.VSTestBridge.UnitTests/Microsoft.Testing.Extensions.VSTestBridge.UnitTests.csproj", + "test/UnitTests/Microsoft.Testing.Extensions.MSBuild.UnitTests/Microsoft.Testing.Extensions.MSBuild.UnitTests.csproj", + "test/UnitTests/Microsoft.Testing.Platform.UnitTests/Microsoft.Testing.Platform.UnitTests.csproj", + "test/UnitTests/MSTest.Analyzers.UnitTests/MSTest.Analyzers.UnitTests.csproj" + ] + } +} \ No newline at end of file diff --git a/eng/Build.props b/eng/Build.props index b8e10fd349..3c39eb0617 100644 --- a/eng/Build.props +++ b/eng/Build.props @@ -11,8 +11,7 @@ - - + From 30eb69819c8c51deb92f6949bc64be224cebc451 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Tue, 18 Mar 2025 14:16:38 +0100 Subject: [PATCH 45/52] Fix --- azure-pipelines.yml | 4 ++-- eng/Build.props | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 40434de131..d372139071 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -163,7 +163,7 @@ stages: # Because the build step is using -ci flag, restore is done in a local .packages directory. # We need to pass NUGET_PACKAGES so that when dotnet test is doing evaluation phase on the projects, it can resolve .props/.targets from packages and import them. # Otherwise, props/targets are not imported. It's important that they are imported so that IsTestingPlatformApplication ends up being set. - - script: $(Build.SourcesDirectory)/.dotnet/dotnet test -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog + - script: $(Build.SourcesDirectory)/.dotnet/dotnet test NonWindowsTests.slnf -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog name: Test displayName: Test env: @@ -220,7 +220,7 @@ stages: # Because the build step is using -ci flag, restore is done in a local .packages directory. # We need to pass NUGET_PACKAGES so that when dotnet test is doing evaluation phase on the projects, it can resolve .props/.targets from packages and import them. # Otherwise, props/targets are not imported. It's important that they are imported so that IsTestingPlatformApplication ends up being set. - - script: $(Build.SourcesDirectory)/.dotnet/dotnet test -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog + - script: $(Build.SourcesDirectory)/.dotnet/dotnet test NonWindowsTests.slnf -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog name: Test displayName: Test env: diff --git a/eng/Build.props b/eng/Build.props index 3c39eb0617..57aac3a459 100644 --- a/eng/Build.props +++ b/eng/Build.props @@ -11,7 +11,11 @@ - + + + + + From 969f68176466c8bfae04357d43e2895155b5ca5d Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Tue, 18 Mar 2025 14:45:55 +0100 Subject: [PATCH 46/52] Fix --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d372139071..a22ff006c7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -163,7 +163,7 @@ stages: # Because the build step is using -ci flag, restore is done in a local .packages directory. # We need to pass NUGET_PACKAGES so that when dotnet test is doing evaluation phase on the projects, it can resolve .props/.targets from packages and import them. # Otherwise, props/targets are not imported. It's important that they are imported so that IsTestingPlatformApplication ends up being set. - - script: $(Build.SourcesDirectory)/.dotnet/dotnet test NonWindowsTests.slnf -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog + - script: $(Build.SourcesDirectory)/.dotnet/dotnet test --solution NonWindowsTests.slnf -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog name: Test displayName: Test env: @@ -220,7 +220,7 @@ stages: # Because the build step is using -ci flag, restore is done in a local .packages directory. # We need to pass NUGET_PACKAGES so that when dotnet test is doing evaluation phase on the projects, it can resolve .props/.targets from packages and import them. # Otherwise, props/targets are not imported. It's important that they are imported so that IsTestingPlatformApplication ends up being set. - - script: $(Build.SourcesDirectory)/.dotnet/dotnet test NonWindowsTests.slnf -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog + - script: $(Build.SourcesDirectory)/.dotnet/dotnet test --solution NonWindowsTests.slnf -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\TestStep.binlog name: Test displayName: Test env: From 0da773e575408e7ce845a437f31b90ad06eb8267 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Tue, 18 Mar 2025 14:58:30 +0100 Subject: [PATCH 47/52] Fix typo --- NonWindowsTests.slnf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NonWindowsTests.slnf b/NonWindowsTests.slnf index c43f27f4bf..e965c3ed7e 100644 --- a/NonWindowsTests.slnf +++ b/NonWindowsTests.slnf @@ -6,8 +6,8 @@ "test/IntegrationTests/MSTest.Acceptance.IntegrationTests/MSTest.Acceptance.IntegrationTests.csproj", "test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Microsoft.Testing.Extensions.UnitTests.csproj", "test/UnitTests/Microsoft.Testing.Extensions.VSTestBridge.UnitTests/Microsoft.Testing.Extensions.VSTestBridge.UnitTests.csproj", - "test/UnitTests/Microsoft.Testing.Extensions.MSBuild.UnitTests/Microsoft.Testing.Extensions.MSBuild.UnitTests.csproj", "test/UnitTests/Microsoft.Testing.Platform.UnitTests/Microsoft.Testing.Platform.UnitTests.csproj", + "test/UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests/Microsoft.Testing.Platform.MSBuild.UnitTests.csproj", "test/UnitTests/MSTest.Analyzers.UnitTests/MSTest.Analyzers.UnitTests.csproj" ] } From 4e75abe3da349426888be96f000e49601c00e125 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Tue, 18 Mar 2025 15:21:59 +0100 Subject: [PATCH 48/52] .NET Framework only on Windows --- .../Microsoft.Testing.Extensions.VSTestBridge.UnitTests.csproj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/UnitTests/Microsoft.Testing.Extensions.VSTestBridge.UnitTests/Microsoft.Testing.Extensions.VSTestBridge.UnitTests.csproj b/test/UnitTests/Microsoft.Testing.Extensions.VSTestBridge.UnitTests/Microsoft.Testing.Extensions.VSTestBridge.UnitTests.csproj index 9d7555a47e..071eb6149a 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.VSTestBridge.UnitTests/Microsoft.Testing.Extensions.VSTestBridge.UnitTests.csproj +++ b/test/UnitTests/Microsoft.Testing.Extensions.VSTestBridge.UnitTests/Microsoft.Testing.Extensions.VSTestBridge.UnitTests.csproj @@ -1,7 +1,8 @@  - $(MicrosoftTestingTargetFrameworks);net462 + $(MicrosoftTestingTargetFrameworks) + $(TargetFrameworks);net462 true Exe From 07daa782a38b2d2e455f872302929db51e9e5988 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Tue, 18 Mar 2025 16:08:55 +0100 Subject: [PATCH 49/52] Fix --- .../Microsoft.Testing.Extensions.UnitTests.csproj | 3 ++- .../Microsoft.Testing.Platform.UnitTests.csproj | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Microsoft.Testing.Extensions.UnitTests.csproj b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Microsoft.Testing.Extensions.UnitTests.csproj index bc3c0aa3a0..9674f0054d 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Microsoft.Testing.Extensions.UnitTests.csproj +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Microsoft.Testing.Extensions.UnitTests.csproj @@ -1,7 +1,8 @@  - $(MicrosoftTestingTargetFrameworks);net462 + $(MicrosoftTestingTargetFrameworks) + $(TargetFrameworks);net462 true Exe diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Microsoft.Testing.Platform.UnitTests.csproj b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Microsoft.Testing.Platform.UnitTests.csproj index 0c5efe56e0..5668eecba6 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Microsoft.Testing.Platform.UnitTests.csproj +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Microsoft.Testing.Platform.UnitTests.csproj @@ -1,7 +1,8 @@  - $(MicrosoftTestingTargetFrameworks);net462 + $(MicrosoftTestingTargetFrameworks) + $(TargetFrameworks);net462 true Exe From c3ae9a3e206c5787293619498dc35304cf66f2df Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Wed, 19 Mar 2025 10:12:06 +0100 Subject: [PATCH 50/52] Ignore flaky --- .../MSTest.Acceptance.IntegrationTests/TimeoutTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutTests.cs index 653c51637c..9c486257be 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutTests.cs @@ -8,6 +8,7 @@ namespace MSTest.Acceptance.IntegrationTests; [TestClass] +[Ignore("These tests are so flaky, which could be related (at least partially) to https://github.com/microsoft/testfx/issues/5165")] public class TimeoutTests : AcceptanceTestBase { private static readonly Dictionary InfoByKind = new() From dacca6a4851d26fc125331438a8f9d19e6c18885 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Thu, 20 Mar 2025 10:48:28 +0100 Subject: [PATCH 51/52] Fix --- .../MSTest.IntegrationTests/OutputTests.cs | 8 ++++---- .../Services/ThreadOperationsTests.cs | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/test/IntegrationTests/MSTest.IntegrationTests/OutputTests.cs b/test/IntegrationTests/MSTest.IntegrationTests/OutputTests.cs index 4a295c78c4..c7720d0655 100644 --- a/test/IntegrationTests/MSTest.IntegrationTests/OutputTests.cs +++ b/test/IntegrationTests/MSTest.IntegrationTests/OutputTests.cs @@ -14,11 +14,11 @@ public class OutputTests : CLITestBase { private const string TestAssetName = "OutputTestProject"; -#if DEBUG - public async Task OutputIsNotMixedWhenTestsRunInParallel() => await ValidateOutputForClassAsync("UnitTest1"); -#endif +#pragma warning disable IDE0051 // Remove unused private members - test is failing in CI. + private async Task OutputIsNotMixedWhenTestsRunInParallel() => await ValidateOutputForClassAsync("UnitTest1"); - public async Task OutputIsNotMixedWhenAsyncTestsRunInParallel() => await ValidateOutputForClassAsync("UnitTest2"); + private async Task OutputIsNotMixedWhenAsyncTestsRunInParallel() => await ValidateOutputForClassAsync("UnitTest2"); +#pragma warning restore IDE0051 // Remove unused private members private static async Task ValidateOutputForClassAsync(string className) { diff --git a/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Services/ThreadOperationsTests.cs b/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Services/ThreadOperationsTests.cs index 9b1b32dcef..030262f265 100644 --- a/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Services/ThreadOperationsTests.cs +++ b/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Services/ThreadOperationsTests.cs @@ -26,7 +26,9 @@ public void ExecuteShouldStartTheActionOnANewThread() } #endif - public void ExecuteShouldReturnFalseIfTheActionTimesOut() +#pragma warning disable IDE0051 // Remove unused private members - test is failing in CI. + private void ExecuteShouldReturnFalseIfTheActionTimesOut() +#pragma warning restore IDE0051 // Remove unused private members { static void Action() => Task.Delay(1000).Wait(); From ce6ae48e2ec57aee3532ef4a40fd74623ad3b817 Mon Sep 17 00:00:00 2001 From: Youssef1313 Date: Thu, 20 Mar 2025 12:44:01 +0100 Subject: [PATCH 52/52] Is it going to be green now? --- samples/Playground/Tests.cs | 3 ++- .../Services/ThreadOperationsTests.cs | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/samples/Playground/Tests.cs b/samples/Playground/Tests.cs index 5baadf9ecc..c3b540e601 100644 --- a/samples/Playground/Tests.cs +++ b/samples/Playground/Tests.cs @@ -15,7 +15,8 @@ public class TestClass [TestMethod] [DynamicData(nameof(Data))] public void Test3(int a, int b) - => throw new Exception("aaaa"); + { + } public static IEnumerable<(int A, int B)> Data { diff --git a/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Services/ThreadOperationsTests.cs b/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Services/ThreadOperationsTests.cs index 030262f265..89456b1846 100644 --- a/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Services/ThreadOperationsTests.cs +++ b/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Services/ThreadOperationsTests.cs @@ -14,8 +14,9 @@ public class ThreadOperationsTests : TestContainer public ThreadOperationsTests() => _asyncOperations = new ThreadOperations(); +#pragma warning disable IDE0051 // Remove unused private members - test is failing in CI. #if NETFRAMEWORK - public void ExecuteShouldStartTheActionOnANewThread() + private void ExecuteShouldStartTheActionOnANewThread() { int actionThreadID = 0; void Action() => actionThreadID = Environment.CurrentManagedThreadId; @@ -26,13 +27,12 @@ public void ExecuteShouldStartTheActionOnANewThread() } #endif -#pragma warning disable IDE0051 // Remove unused private members - test is failing in CI. private void ExecuteShouldReturnFalseIfTheActionTimesOut() -#pragma warning restore IDE0051 // Remove unused private members { static void Action() => Task.Delay(1000).Wait(); CancellationTokenSource tokenSource = new(); Verify(!_asyncOperations.Execute(Action, 1, tokenSource.Token)); } +#pragma warning restore IDE0051 // Remove unused private members }