diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 20a6e88ffe..19a4712c00 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -42,7 +42,7 @@ { "name": "dotnet-test", "source": "./plugins/dotnet-test", - "description": "Skills for running and diagnosing .NET tests: test execution, filtering, platform detection, and MSTest workflows." + "description": "Skills for running, diagnosing, and migrating .NET tests: test execution, filtering, platform detection, and MSTest workflows." } ] } diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json index 20a6e88ffe..19a4712c00 100644 --- a/.github/plugin/marketplace.json +++ b/.github/plugin/marketplace.json @@ -42,7 +42,7 @@ { "name": "dotnet-test", "source": "./plugins/dotnet-test", - "description": "Skills for running and diagnosing .NET tests: test execution, filtering, platform detection, and MSTest workflows." + "description": "Skills for running, diagnosing, and migrating .NET tests: test execution, filtering, platform detection, and MSTest workflows." } ] } diff --git a/README.md b/README.md index 2d8a58119b..239c08b570 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ This repository contains the .NET team's curated set of core skills and custom a | [dotnet-upgrade](plugins/dotnet-upgrade/) | Skills for migrating and upgrading .NET projects across framework versions, language features, and compatibility targets. | | [dotnet-maui](plugins/dotnet-maui/) | Skills for .NET MAUI development: environment setup, diagnostics, and troubleshooting. | | [dotnet-ai](plugins/dotnet-ai/) | AI and ML skills for .NET: technology selection, LLM integration, agentic workflows, RAG pipelines, MCP, and classic ML with ML.NET. | -| [dotnet-test](plugins/dotnet-test/) | Skills for running and diagnosing .NET tests: test execution, filtering, platform detection, and MSTest workflows. | +| [dotnet-test](plugins/dotnet-test/) | Skills for running, diagnosing, and migrating .NET tests: test execution, filtering, platform detection, and MSTest workflows. | ## Installation diff --git a/plugins/dotnet-test/plugin.json b/plugins/dotnet-test/plugin.json index bd47c0eabb..e270b4eea3 100644 --- a/plugins/dotnet-test/plugin.json +++ b/plugins/dotnet-test/plugin.json @@ -1,6 +1,6 @@ { "name": "dotnet-test", "version": "0.1.0", - "description": "Skills for running and diagnosing .NET tests: test execution, filtering, platform detection, and MSTest workflows.", + "description": "Skills for running, diagnosing, and migrating .NET tests: test execution, filtering, platform detection, and MSTest workflows.", "skills": "./skills/" } diff --git a/plugins/dotnet-test/skills/migrate-vstest-to-mtp/SKILL.md b/plugins/dotnet-test/skills/migrate-vstest-to-mtp/SKILL.md new file mode 100644 index 0000000000..7161d3ec2f --- /dev/null +++ b/plugins/dotnet-test/skills/migrate-vstest-to-mtp/SKILL.md @@ -0,0 +1,332 @@ +--- +name: migrate-vstest-to-mtp +description: > + Migrates .NET test projects from VSTest to Microsoft.Testing.Platform (MTP). + Use when user asks to "migrate to MTP", "switch from VSTest", "enable + Microsoft.Testing.Platform", "use MTP runner", or mentions EnableMSTestRunner, + EnableNUnitRunner, UseMicrosoftTestingPlatformRunner, or dotnet test exit + code 8. Supports MSTest, NUnit, xUnit.net v2 (via YTest.MTP.XUnit2), and + xUnit.net v3 (native MTP). Also covers translating xUnit.net v3 MTP filter + syntax (--filter-class, --filter-trait, --filter-query). + Covers runner enablement, CLI argument translation, Directory.Build.props + and global.json configuration, CI/CD pipeline updates, and MTP extension + packages. DO NOT USE FOR: migrating between test frameworks + (MSTest/xUnit/NUnit), xUnit.net v2 to v3 API migration, MSTest version + upgrades (use migrate-mstest-* skills), TFM upgrades, or UWP/WinUI test + projects. +--- + +# VSTest → Microsoft.Testing.Platform Migration + +Migrate a .NET test solution from VSTest to Microsoft.Testing.Platform (MTP). The outcome is a solution where all test projects run on MTP, `dotnet test` works correctly, and CI/CD pipelines are updated. + +> **Important**: Do not mix VSTest-based and MTP-based .NET test projects in the same solution or run configuration — this is an unsupported scenario. + +## When to Use + +- Switching from VSTest to Microsoft.Testing.Platform for any supported test framework +- Enabling `dotnet run` / `dotnet watch` / direct executable execution for test projects +- Enabling Native AOT or trimmed test execution +- Replacing `vstest.console.exe` with `dotnet test` on MTP +- Updating CI/CD pipelines from the VSTest task to the .NET Core CLI task +- Updating `dotnet test` arguments from VSTest syntax to MTP syntax + +## When Not to Use + +- The project already runs on Microsoft.Testing.Platform — migration is done +- Migrating between test frameworks (e.g., MSTest to xUnit.net) — different effort entirely +- The project builds UWP or packaged WinUI test projects — MTP does not support these yet +- The solution mixes .NET and non-.NET test adapters (e.g., JavaScript or C++ adapters) — VSTest is required +- Upgrading MSTest versions — use `migrate-mstest-v1v2-to-v3` or `migrate-mstest-v3-to-v4` + +## Workflow + +### Step 1: Assess the solution + +1. Identify the test framework for each test project: + - **MSTest**: References `MSTest` or `MSTest.TestAdapter`, or uses `MSTest.Sdk` (with `` not set to `false`). Note: `MSTest.TestFramework` alone is a library dependency, not a test project. + - **NUnit**: References `NUnit3TestAdapter` + - **xUnit.net**: References `xunit` and `xunit.runner.visualstudio` +2. Check the .NET SDK version (`dotnet --version`) — this determines how `dotnet test` integrates with MTP +3. Check whether a `Directory.Build.props` file exists at the solution or repo root — all MTP properties should go there for consistency +4. Check for `vstest.console.exe` usage in CI scripts or pipeline definitions +5. Check for VSTest-specific `dotnet test` arguments in CI scripts: `--filter`, `--logger`, `--collect`, `--settings`, `--blame*` +6. Run `dotnet test` to establish a baseline of test pass/fail counts + +### Step 2: Set up Directory.Build.props + +> **Critical**: Always set MTP properties in `Directory.Build.props` at the solution or repo root — never per-project. This prevents inconsistent configuration where some projects use VSTest and others use MTP (an unsupported scenario). + +> **Note**: MTP requires test projects to have `Exe`. Only `MSTest.Sdk` sets this automatically. For all other setups (MSTest NuGet packages with `EnableMSTestRunner`, NUnit with `EnableNUnitRunner`, xUnit.net with `YTest.MTP.XUnit2`), you must set `Exe` explicitly — either per-project or in `Directory.Build.props` with a condition that targets only test projects. + +### Step 3: Enable the framework-specific MTP runner + +Each framework has its own opt-in property. Add these in `Directory.Build.props` for consistency. + +#### MSTest + +**Option A — MSTest NuGet packages (3.2.0+):** + +```xml + + true + Exe + +``` + +Ensure the project references MSTest 3.2.0 or later. If the version is already 3.2.0+, no MSTest version upgrade is needed for MTP migration. + +**Option B — MSTest.Sdk:** + +When using `MSTest.Sdk`, MTP is enabled by default — no `EnableMSTestRunner` or `OutputType Exe` property is needed (the SDK sets both automatically). The only action is: if the project has `true`, **remove it**. That property forces the project to use VSTest instead of MTP. + +#### NUnit + +Requires `NUnit3TestAdapter` **5.0.0** or later. + +1. Update `NUnit3TestAdapter` to 5.0.0+: + +```xml + +``` + +2. Enable the NUnit runner: + +```xml + + true + Exe + +``` + +#### xUnit.net + +Add a reference to `YTest.MTP.XUnit2` — this package provides MTP support for xUnit.net v2 projects without requiring an upgrade to xunit.v3. You must also set `OutputType` to `Exe`: + +```xml + +``` + +```xml + + Exe + +``` + +> **Note**: `YTest.MTP.XUnit2` preserves the VSTest `--filter` syntax, so no filter migration is needed for xUnit.net v2. It also supports `--settings` for runsettings (xunit-specific configurations only), `xunit.runner.json`, TRX reporting via `--report-trx`, and `--treenode-filter`. + +#### xUnit.net v3 + +xUnit.net v3 (`xunit.v3` package) has built-in MTP support. Enable it with: + +```xml + + true + +``` + +> **Important**: xUnit.net v3 on MTP does NOT support the VSTest `--filter` syntax. You must translate filters to xUnit.net v3's native filter options (see Step 5). + +### Step 4: Configure dotnet test integration + +The `dotnet test` integration depends on the .NET SDK version. + +#### .NET 10 SDK and later (recommended) + +Use the native MTP mode by adding a `test` section to `global.json`: + +```json +{ + "sdk": { + "version": "10.0.100" + }, + "test": { + "runner": "Microsoft.Testing.Platform" + } +} +``` + +In this mode, `dotnet test` arguments are passed directly — for example, `dotnet test --report-trx`. + +> **Important**: `global.json` does not support trailing commas. Ensure the JSON is strictly valid. + +#### .NET 9 SDK and earlier + +Use the VSTest mode of `dotnet test` command to run MTP test projects by adding this property in `Directory.Build.props`: + +```xml + + true + +``` + +> **Important**: In this mode, you must use `--` to separate `dotnet test` build arguments from MTP arguments. For example: `dotnet test --no-build -- --list-tests`. + +### Step 5: Update dotnet test command-line arguments + +VSTest-specific arguments must be translated to MTP equivalents. Build-related arguments (`-c`, `-f`, `--no-build`, `--nologo`, `-v`, etc.) are unchanged. + +| VSTest argument | MTP equivalent | Notes | +|-----------------|----------------|-------| +| `--test-adapter-path` | Not applicable | MTP does not use external adapter discovery | +| `--blame` | Not applicable | | +| `--blame-crash` | `--crashdump` | Requires `Microsoft.Testing.Extensions.CrashDump` NuGet package | +| `--blame-crash-dump-type ` | `--crashdump-type ` | Requires CrashDump extension | +| `--blame-hang` | `--hangdump` | Requires `Microsoft.Testing.Extensions.HangDump` NuGet package | +| `--blame-hang-dump-type ` | `--hangdump-type ` | Requires HangDump extension | +| `--blame-hang-timeout ` | `--hangdump-timeout ` | Requires HangDump extension | +| `--collect "Code Coverage;Format=cobertura"` | `--coverage --coverage-output-format cobertura` | Per-extension arguments | +| `-d\|--diag ` | `--diagnostic` | | +| `--filter ` | `--filter ` | Same syntax for MSTest, NUnit, and xUnit.net v2 (with `YTest.MTP.XUnit2`). For xUnit.net v3, see filter migration below | +| `-l\|--logger trx` | `--report-trx` | Requires `Microsoft.Testing.Extensions.TrxReport` NuGet package | +| `--results-directory ` | `--results-directory ` | Same | +| `-s\|--settings ` | `--settings ` | MSTest and NUnit still support `.runsettings` | +| `-t\|--list-tests` | `--list-tests` | Same | +| `-- ` | `--test-parameter` | Applicable only to MSTest and NUnit | + +#### Filter migration + +**MSTest, NUnit, and xUnit.net v2 (with `YTest.MTP.XUnit2`)**: The VSTest `--filter` syntax is identical on both VSTest and MTP. No changes needed. + +**xUnit.net v3 (native MTP)**: xUnit.net v3 does NOT support the VSTest `--filter` syntax on MTP. Translate filters using xUnit.net v3's native options: + +| VSTest `--filter` syntax | xUnit.net v3 MTP equivalent | Notes | +|---|---|---| +| `FullyQualifiedName~ClassName` | `--filter-class *ClassName*` | Wildcards required for substring match | +| `FullyQualifiedName=Ns.Class.Method` | `--filter-method Ns.Class.Method` | Exact match on fully qualified method | +| `Name=MethodName` | `--filter-method *MethodName*` | Wildcards for substring match | +| `Category=Value` (trait) | `--filter-trait "Category=Value"` | Filter by trait name/value pair | +| Complex expressions | `--filter-query "expr"` | Uses [xUnit.net query filter language](https://xunit.net/docs/query-filter-language) | + +`--filter-query` uses a path-segment syntax with four segments and an optional trait filter: + +``` +////[traitName=traitValue] +``` + +Each segment matches against: assembly name, namespace, class name, method name. Use `*` for "match all" in any segment. Documentation: https://xunit.net/docs/query-filter-language + +```shell +# VSTest +dotnet test --filter "FullyQualifiedName~IntegrationTests&Category=Smoke" + +# xUnit.net v3 MTP — using individual filters (AND behavior) +dotnet test -- --filter-class *IntegrationTests* --filter-trait "Category=Smoke" + +# xUnit.net v3 MTP — using query language (assembly/namespace/class/method[trait]) +dotnet test -- --filter-query "/*/*/*IntegrationTests*/*[Category=Smoke]" +``` + +> **Note**: When combining `--filter-class` and `--filter-trait`, both conditions must match (AND behavior). For complex expressions, use `--filter-query` with the path-segment syntax. See the [xUnit.net query filter language docs](https://xunit.net/docs/query-filter-language) for full reference. + +### Step 6: Install MTP extension packages (if needed) + +If CI scripts use TRX reporting, crash dumps, or hang dumps, add the corresponding NuGet packages: + +```xml + + + + + + + + + + + +``` + +### Step 7: Update CI/CD pipelines + +#### Azure DevOps + +**If using the VSTest task (`VSTest@3`)**: Replace with the .NET Core CLI task (`DotNetCoreCLI@2`): + +```yaml +# Before (VSTest task) +- task: VSTest@3 + inputs: + testAssemblyVer2: '**/*Tests.dll' + runSettingsFile: 'test.runsettings' + +# After (.NET Core CLI task) +- task: DotNetCoreCLI@2 + displayName: Run tests + inputs: + command: 'test' + arguments: '--no-build --configuration Release' +``` + +**If already using DotNetCoreCLI@2**: Update arguments per Step 5 translations. Remember the `--` separator on .NET 9 and earlier: + +```yaml +- task: DotNetCoreCLI@2 + displayName: Run tests + inputs: + command: 'test' + arguments: '--no-build -- --report-trx --results-directory $(Agent.TempDirectory)' +``` + +#### GitHub Actions + +Update `dotnet test` invocations in workflow files with the same argument translations from Step 5. + +#### Replace vstest.console.exe + +If any script invokes `vstest.console.exe` directly, replace it with `dotnet test`. The test projects are now executables and can also be run directly. + +### Step 8: Handle behavioral differences + +#### Zero tests exit code + +VSTest silently succeeds when zero tests are discovered. MTP fails with **exit code 8**. Options: + +- Pass `--ignore-exit-code 8` when running tests +- Add to `Directory.Build.props`: + +```xml + + $(TestingPlatformCommandLineArguments) --ignore-exit-code 8 + +``` + +- Use environment variable: `TESTINGPLATFORM_EXITCODE_IGNORE=8` + +### Step 9: Remove VSTest-only packages (optional) + +Once migration is complete and verified, remove packages that are only needed for VSTest: + +- `Microsoft.NET.Test.Sdk` — not needed for MTP (MSTest.Sdk v4 already omits it by default) +- `xunit.runner.visualstudio` — only needed for VSTest discovery of xUnit.net (not needed when using `YTest.MTP.XUnit2`) +- `NUnit3TestAdapter` VSTest-only features — the adapter is still needed but only for the MTP runner + +> **Note**: If you need to maintain VSTest compatibility during a transition period, keep these packages. + +### Step 10: Verify + +1. Run `dotnet build` — confirm zero errors +2. Run `dotnet test` — confirm all tests pass +3. Compare test pass/fail counts to the pre-migration baseline +4. Run the test executable directly (e.g., `./bin/Debug/net8.0/MyTests.exe`) — confirm it works +5. Verify CI pipeline produces the expected test result artifacts (TRX files, code coverage, crash dumps) +6. Test that Test Explorer in Visual Studio (17.14+) or VS Code discovers and runs tests + +## Common Pitfalls + +| Pitfall | Solution | +|---------|----------| +| Mixing VSTest and MTP projects in the same solution | Migrate all test projects together — mixed mode is unsupported | +| `dotnet test` arguments ignored on .NET 9 and earlier | Use `--` to separate build args from MTP args: `dotnet test -- --report-trx` | +| Exit code 8 on CI without failures | MTP fails when zero tests run; use `--ignore-exit-code 8` or fix test discovery | +| MSTest.Sdk v4 + vstest.console no longer works | MSTest.Sdk v4 no longer adds `Microsoft.NET.Test.Sdk` — add it explicitly or switch to `dotnet test` | +| Missing `Exe` | Required for all setups except MSTest.Sdk (which sets it automatically) | + +## More Info + +- [Test platforms overview](https://learn.microsoft.com/dotnet/core/testing/test-platforms-overview) +- [Migrate from VSTest to Microsoft.Testing.Platform](https://learn.microsoft.com/dotnet/core/testing/migrating-vstest-microsoft-testing-platform) +- [Microsoft.Testing.Platform overview](https://learn.microsoft.com/dotnet/core/testing/microsoft-testing-platform-intro) +- [Testing with dotnet test](https://learn.microsoft.com/dotnet/core/testing/unit-testing-with-dotnet-test) +- [Microsoft.Testing.Platform CLI options](https://learn.microsoft.com/dotnet/core/testing/microsoft-testing-platform-cli-options) +- [Microsoft.Testing.Platform extensions](https://learn.microsoft.com/dotnet/core/testing/unit-testing-platform-extensions) diff --git a/tests/dotnet-test/migrate-vstest-to-mtp/eval.yaml b/tests/dotnet-test/migrate-vstest-to-mtp/eval.yaml new file mode 100644 index 0000000000..36e0113849 --- /dev/null +++ b/tests/dotnet-test/migrate-vstest-to-mtp/eval.yaml @@ -0,0 +1,371 @@ +scenarios: + # ============================================================================ + # Goal 1: Migrate MSTest project from VSTest to MTP + # ============================================================================ + + - name: "Migrate MSTest project from VSTest to Microsoft.Testing.Platform" + prompt: | + I have a solution with 3 MSTest 3.8.0 test projects that currently run on VSTest. + I want to switch all of them to Microsoft.Testing.Platform. We're on .NET 8 SDK + and use `dotnet test --logger trx` in CI. What changes do I need to make to each + project and to the CI command? I want properties set consistently across all projects. + setup: + files: + - path: "TestProject.csproj" + source: "../../../../tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-vstest/TestProject.csproj" + - path: "CalculatorTests.cs" + source: "../../../../tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-vstest/CalculatorTests.cs" + assertions: + - type: "output_matches" + pattern: "(EnableMSTestRunner|MSTest\\.Sdk)" + - type: "output_matches" + pattern: "(OutputType|Exe)" + - type: "output_matches" + pattern: "(Directory\\.Build\\.props|centralize|shared)" + rubric: + - "Adds true to Directory.Build.props (not per-project)" + - "Mentions Exe must be set explicitly on test projects (EnableMSTestRunner does NOT auto-set it)" + - "Configures dotnet test integration (TestingPlatformDotnetTestSupport in Directory.Build.props or global.json)" + - "Explains that --logger trx must change to --report-trx and requires Microsoft.Testing.Extensions.TrxReport package" + - "Mentions the -- separator requirement on .NET 8 for MTP arguments" + - "Does not suggest upgrading MSTest version (3.2.0+ is sufficient, 3.8.0 qualifies)" + timeout: 360 + + # ============================================================================ + # Goal 2: Migrate NUnit project from VSTest to MTP + # ============================================================================ + + - name: "Migrate NUnit project from VSTest to Microsoft.Testing.Platform" + prompt: | + My NUnit test project uses NUnit3TestAdapter 4.6.0 and Microsoft.NET.Test.Sdk + on VSTest. I want to migrate to Microsoft.Testing.Platform. What do I need to change? + setup: + files: + - path: "TestProject.csproj" + source: "../../../../tests/dotnet-test/migrate-vstest-to-mtp/fixtures/nunit-vstest/TestProject.csproj" + - path: "CalculatorTests.cs" + source: "../../../../tests/dotnet-test/migrate-vstest-to-mtp/fixtures/nunit-vstest/CalculatorTests.cs" + assertions: + - type: "output_matches" + pattern: "(EnableNUnitRunner|NUnit.*runner)" + - type: "output_matches" + pattern: "(NUnit3TestAdapter|5\\.0\\.0|adapter.*5)" + - type: "output_matches" + pattern: "(OutputType|Exe)" + rubric: + - "Identifies that NUnit3TestAdapter must be upgraded to 5.0.0 or later for MTP support" + - "Adds true" + - "Adds Exe" + - "Mentions configuring dotnet test integration" + - "Recommends using Directory.Build.props for MTP properties" + timeout: 360 + + # ============================================================================ + # Goal 3: Migrate xUnit.net project from VSTest to MTP + # ============================================================================ + + - name: "Migrate xUnit.net v2 project from VSTest to Microsoft.Testing.Platform" + prompt: | + My xUnit.net project uses xunit 2.9.3 with xunit.runner.visualstudio on VSTest. + I want to switch to Microsoft.Testing.Platform. What should I do? + setup: + files: + - path: "TestProject.csproj" + source: "../../../../tests/dotnet-test/migrate-vstest-to-mtp/fixtures/xunit-vstest/TestProject.csproj" + - path: "CalculatorTests.cs" + source: "../../../../tests/dotnet-test/migrate-vstest-to-mtp/fixtures/xunit-vstest/CalculatorTests.cs" + assertions: + - type: "output_matches" + pattern: "(YTest\\.MTP\\.XUnit2|MTP.*xunit)" + - type: "output_matches" + pattern: "(PackageReference|package)" + rubric: + - "Recommends adding the YTest.MTP.XUnit2 package to enable MTP support for xUnit.net v2" + - "Does NOT require upgrading to xunit.v3 — YTest.MTP.XUnit2 works with xUnit.net v2" + - "Mentions that VSTest --filter syntax is preserved with YTest.MTP.XUnit2 (no filter migration needed)" + - "Mentions configuring dotnet test integration (TestingPlatformDotnetTestSupport or global.json)" + - "Optionally mentions that xunit.runner.visualstudio and Microsoft.NET.Test.Sdk can be removed once VSTest compat is no longer needed" + timeout: 360 + + # ============================================================================ + # Goal 4: Update Azure DevOps CI pipeline for MTP + # ============================================================================ + + - name: "Update Azure DevOps pipeline from VSTest task to MTP" + prompt: | + Our CI pipeline uses the VSTest@3 task to run MSTest tests. We're migrating + to Microsoft.Testing.Platform. We also produce TRX reports and collect code + coverage. How do we update the pipeline? + setup: + files: + - path: "TestProject.csproj" + source: "../../../../tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-ci-pipeline/TestProject.csproj" + - path: "ServiceTests.cs" + source: "../../../../tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-ci-pipeline/ServiceTests.cs" + - path: "azure-pipelines.yml" + source: "../../../../tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-ci-pipeline/azure-pipelines.yml" + assertions: + - type: "output_matches" + pattern: "(DotNetCoreCLI|dotnet test)" + - type: "output_matches" + pattern: "(VSTest@3|replace|remove)" + - type: "output_matches" + pattern: "(report-trx|TrxReport)" + rubric: + - "Identifies that VSTest@3 task must be replaced with DotNetCoreCLI@2 task" + - "Shows the updated pipeline YAML using DotNetCoreCLI@2 with command 'test'" + - "Translates --logger trx to --report-trx and mentions Microsoft.Testing.Extensions.TrxReport package" + - "Translates code coverage collection to --coverage and mentions CodeCoverage extension package" + - "Explains the -- separator needed for MTP arguments on .NET 9 and earlier" + timeout: 300 + + # ============================================================================ + # Goal 5: MSTest.Sdk project with UseVSTest + # ============================================================================ + + - name: "Migrate MSTest.Sdk project that explicitly uses VSTest" + prompt: | + My project uses MSTest.Sdk/3.8.0 with true. I want + to switch to Microsoft.Testing.Platform. What do I need to change? + setup: + files: + - path: "TestProject.csproj" + source: "../../../../tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-sdk-vstest/TestProject.csproj" + - path: "AppTests.cs" + source: "../../../../tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-sdk-vstest/AppTests.cs" + assertions: + - type: "output_matches" + pattern: "(UseVSTest|remove|delete)" + - type: "output_matches" + pattern: "(MSTest\\.Sdk|MTP.*default)" + rubric: + - "Identifies that MSTest.Sdk uses MTP by default and UseVSTest opts out of it" + - "Recommends removing true to enable MTP" + - "Notes that MSTest.Sdk automatically sets EnableMSTestRunner and OutputType Exe — no additional properties needed" + - "Mentions configuring dotnet test integration (TestingPlatformDotnetTestSupport or global.json)" + timeout: 360 + + # ============================================================================ + # Goal 6: Translate dotnet test arguments + # ============================================================================ + + - name: "Translate dotnet test VSTest arguments to MTP equivalents" + prompt: | + Our CI runs `dotnet test --filter "Category=Integration" --logger trx --collect "Code Coverage;Format=cobertura" --blame-crash --blame-hang --blame-hang-timeout 5m`. + We're switching to Microsoft.Testing.Platform on .NET 8. What should the + new command look like? + setup: + files: + - path: "TestProject.csproj" + source: "../../../../tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-vstest/TestProject.csproj" + - path: "CalculatorTests.cs" + source: "../../../../tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-vstest/CalculatorTests.cs" + assertions: + - type: "output_matches" + pattern: "(report-trx|--report-trx)" + - type: "output_matches" + pattern: "(--coverage|CodeCoverage)" + - type: "output_matches" + pattern: "(crashdump|--crashdump)" + - type: "output_matches" + pattern: "(hangdump|--hangdump)" + rubric: + - "Translates --logger trx to --report-trx" + - "Translates --collect \"Code Coverage;Format=cobertura\" to --coverage --coverage-output-format cobertura" + - "Translates --blame-crash to --crashdump" + - "Translates --blame-hang to --hangdump" + - "Translates --blame-hang-timeout 5m to --hangdump-timeout 5m" + - "Mentions the -- separator needed on .NET 8 (pre-.NET 10)" + - "Lists the required NuGet extension packages (TrxReport, CrashDump, HangDump, CodeCoverage)" + timeout: 300 + + # ============================================================================ + # Goal 7: Handle zero-test exit code behavioral change + # ============================================================================ + + - name: "Handle exit code 8 when migrating from VSTest to MTP" + prompt: | + We migrated our test project from VSTest to Microsoft.Testing.Platform. + Now our CI pipeline fails with exit code 8 even though no tests actually + failed. Some test assemblies have conditional compilation and produce zero + tests on certain configurations. With VSTest this was fine. How do I fix + this behavioral difference? + setup: + files: + - path: "TestProject.csproj" + content: | + + + net8.0 + Exe + true + true + + + + + + - path: "ConditionalTests.cs" + content: | + using Microsoft.VisualStudio.TestTools.UnitTesting; + + namespace MyApp.Tests; + + #if INCLUDE_SLOW_TESTS + [TestClass] + public class SlowIntegrationTests + { + [TestMethod] + public void LongRunning_Succeeds() + { + Assert.IsTrue(true); + } + } + #endif + assertions: + - type: "output_matches" + pattern: "(exit.?code.?8|ignore-exit-code|zero.?test)" + rubric: + - "Explains that MTP fails with exit code 8 when zero tests are discovered (unlike VSTest which succeeds)" + - "Recommends --ignore-exit-code 8 as a command-line option" + - "Shows how to set it via TestingPlatformCommandLineArguments MSBuild property" + - "Mentions TESTINGPLATFORM_EXITCODE_IGNORE environment variable as an alternative" + timeout: 360 + + # ============================================================================ + # Goal 8: Configure global.json for .NET 10 native MTP mode + # ============================================================================ + + - name: "Configure dotnet test MTP mode on .NET 10 SDK" + prompt: | + We're on .NET 10 SDK and migrating to Microsoft.Testing.Platform. I read + that .NET 10 has native MTP support in dotnet test. How do I set it up? + Do I still need TestingPlatformDotnetTestSupport? + setup: + files: + - path: "TestProject.csproj" + content: | + + + net10.0 + Exe + true + true + + + + + + - path: "global.json" + content: | + { + "sdk": { + "version": "10.0.100" + } + } + assertions: + - type: "output_matches" + pattern: "(global\\.json|test.*runner)" + - type: "output_matches" + pattern: "(Microsoft\\.Testing\\.Platform|MTP)" + rubric: + - "Shows how to add the test.runner section to global.json" + - "Explains that TestingPlatformDotnetTestSupport is no longer needed on .NET 10 with native MTP mode" + - "Explains that the -- separator is no longer needed in .NET 10 native MTP mode" + - "Recommends using the native MTP mode for .NET 10+ instead of the VSTest bridge" + timeout: 240 + + # ============================================================================ + # Goal 9: xUnit.net filter migration + # ============================================================================ + + - name: "Migrate xUnit.net VSTest filter syntax to MTP" + prompt: | + My CI uses `dotnet test --filter "FullyQualifiedName~IntegrationTests&Category=Smoke"` + with xUnit.net v2. I'm upgrading to xunit.v3 with MTP. The --filter no longer + works the same way. How do I translate the filter? + setup: + files: + - path: "TestProject.csproj" + content: | + + + net8.0 + Exe + true + + + + + + - path: "IntegrationTests.cs" + content: | + using Xunit; + + namespace MyApp.Tests; + + [Trait("Category", "Smoke")] + public class IntegrationTests + { + [Fact] + public void HealthEndpoint_ReturnsOk() + { + Assert.True(true); + } + + [Theory] + [InlineData("api/v1")] + [InlineData("api/v2")] + public void ApiEndpoint_ReturnsOk(string path) + { + Assert.False(string.IsNullOrEmpty(path)); + } + } + assertions: + - type: "output_matches" + pattern: "(filter-class|filter-trait|filter-method|filter-query)" + rubric: + - "Explains that VSTest --filter syntax is not supported for xUnit.net v3 on MTP" + - "Translates FullyQualifiedName~IntegrationTests to --filter-class *IntegrationTests* (with wildcards for substring match) or --filter-query with path-segment syntax" + - "Translates Category=Smoke to --filter-trait \"Category=Smoke\"" + - "Mentions --filter-query uses path-segment syntax (////[trait=value])" + - "References xUnit.net query filter language documentation or provides the link" + timeout: 360 + + # ============================================================================ + # Goal 10: Full end-to-end migration with multiple projects + # ============================================================================ + + - name: "Full VSTest to MTP migration plan for MSTest solution" + prompt: | + I have a solution with 5 MSTest test projects all using VSTest (MSTest.TestFramework 3.8.0, + MSTest.TestAdapter 3.8.0, Microsoft.NET.Test.Sdk 17.13.0). We use Azure DevOps + with the VSTest@3 task, TRX reporting, and code coverage. We're on .NET 8 SDK. + I want a complete migration plan to Microsoft.Testing.Platform. + setup: + files: + - path: "TestProject.csproj" + source: "../../../../tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-vstest/TestProject.csproj" + - path: "CalculatorTests.cs" + source: "../../../../tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-vstest/CalculatorTests.cs" + assertions: + - type: "output_matches" + pattern: "(EnableMSTestRunner|MSTest.*runner)" + - type: "output_matches" + pattern: "(OutputType|Exe)" + - type: "output_matches" + pattern: "(Directory\\.Build\\.props|centralize)" + - type: "output_matches" + pattern: "(DotNetCoreCLI|VSTest@3|pipeline)" + - type: "output_matches" + pattern: "(TrxReport|report-trx)" + rubric: + - "Recommends creating or updating Directory.Build.props with OutputType, EnableMSTestRunner, and TestingPlatformDotnetTestSupport" + - "Explains the properties should be set centrally, not per-project" + - "Covers replacing VSTest@3 task with DotNetCoreCLI@2" + - "Covers TRX reporting migration (--report-trx and TrxReport NuGet package)" + - "Covers code coverage migration (--coverage and CodeCoverage NuGet package)" + - "Mentions the -- separator requirement on .NET 8" + - "Includes verification steps (build, test, compare counts, check CI artifacts)" + - "Mentions possibility of exit code 8 for zero-test assemblies" + timeout: 300 diff --git a/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-ci-pipeline/ServiceTests.cs b/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-ci-pipeline/ServiceTests.cs new file mode 100644 index 0000000000..542a31e711 --- /dev/null +++ b/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-ci-pipeline/ServiceTests.cs @@ -0,0 +1,19 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace MyApp.Tests; + +[TestClass] +public class ServiceTests +{ + [TestMethod] + public void HealthCheck_ReturnsOk() + { + Assert.IsTrue(true); + } + + [TestMethod] + public void GetStatus_ReturnsRunning() + { + Assert.AreEqual("Running", "Running"); + } +} diff --git a/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-ci-pipeline/TestProject.csproj b/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-ci-pipeline/TestProject.csproj new file mode 100644 index 0000000000..8948a2db6b --- /dev/null +++ b/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-ci-pipeline/TestProject.csproj @@ -0,0 +1,13 @@ + + + net8.0 + enable + enable + false + + + + + + + diff --git a/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-ci-pipeline/azure-pipelines.yml b/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-ci-pipeline/azure-pipelines.yml new file mode 100644 index 0000000000..c7089e55ee --- /dev/null +++ b/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-ci-pipeline/azure-pipelines.yml @@ -0,0 +1,36 @@ +trigger: + - main + +pool: + vmImage: 'ubuntu-latest' + +steps: + - task: UseDotNet@2 + inputs: + packageType: 'sdk' + version: '8.0.x' + + - task: DotNetCoreCLI@2 + displayName: Restore + inputs: + command: 'restore' + + - task: DotNetCoreCLI@2 + displayName: Build + inputs: + command: 'build' + arguments: '--no-restore --configuration Release' + + - task: VSTest@3 + displayName: Run tests + inputs: + testAssemblyVer2: | + **/*Tests.dll + !**/obj/** + runSettingsFile: 'test.runsettings' + codeCoverageEnabled: true + + - task: PublishTestResults@2 + inputs: + testResultsFormat: 'VSTest' + testResultsFiles: '**/*.trx' diff --git a/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-sdk-vstest/AppTests.cs b/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-sdk-vstest/AppTests.cs new file mode 100644 index 0000000000..d1ee2c6bb9 --- /dev/null +++ b/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-sdk-vstest/AppTests.cs @@ -0,0 +1,19 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace MyApp.Tests; + +[TestClass] +public class AppTests +{ + [TestMethod] + public void Initialize_Succeeds() + { + Assert.IsTrue(true); + } + + [TestMethod] + public void Shutdown_Succeeds() + { + Assert.IsTrue(true); + } +} diff --git a/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-sdk-vstest/TestProject.csproj b/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-sdk-vstest/TestProject.csproj new file mode 100644 index 0000000000..9a3a2041b0 --- /dev/null +++ b/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-sdk-vstest/TestProject.csproj @@ -0,0 +1,8 @@ + + + net8.0 + enable + enable + true + + diff --git a/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-vstest/CalculatorTests.cs b/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-vstest/CalculatorTests.cs new file mode 100644 index 0000000000..f85911b61e --- /dev/null +++ b/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-vstest/CalculatorTests.cs @@ -0,0 +1,26 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace MyApp.Tests; + +[TestClass] +public class CalculatorTests +{ + [TestMethod] + public void Add_ReturnsCorrectSum() + { + Assert.AreEqual(4, 2 + 2); + } + + [TestMethod] + public void Subtract_ReturnsCorrectDifference() + { + Assert.AreEqual(0, 2 - 2); + } + + [TestMethod] + [TestCategory("Slow")] + public void Multiply_ReturnsCorrectProduct() + { + Assert.AreEqual(6, 2 * 3); + } +} diff --git a/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-vstest/TestProject.csproj b/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-vstest/TestProject.csproj new file mode 100644 index 0000000000..8948a2db6b --- /dev/null +++ b/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/mstest-vstest/TestProject.csproj @@ -0,0 +1,13 @@ + + + net8.0 + enable + enable + false + + + + + + + diff --git a/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/nunit-vstest/CalculatorTests.cs b/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/nunit-vstest/CalculatorTests.cs new file mode 100644 index 0000000000..a5afea22c8 --- /dev/null +++ b/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/nunit-vstest/CalculatorTests.cs @@ -0,0 +1,26 @@ +using NUnit.Framework; + +namespace MyApp.Tests; + +[TestFixture] +public class CalculatorTests +{ + [Test] + public void Add_ReturnsCorrectSum() + { + Assert.That(2 + 2, Is.EqualTo(4)); + } + + [Test] + public void Subtract_ReturnsCorrectDifference() + { + Assert.That(2 - 2, Is.EqualTo(0)); + } + + [Test] + [Category("Slow")] + public void Multiply_ReturnsCorrectProduct() + { + Assert.That(2 * 3, Is.EqualTo(6)); + } +} diff --git a/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/nunit-vstest/TestProject.csproj b/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/nunit-vstest/TestProject.csproj new file mode 100644 index 0000000000..b40618c0ec --- /dev/null +++ b/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/nunit-vstest/TestProject.csproj @@ -0,0 +1,14 @@ + + + net8.0 + enable + enable + false + + + + + + + + diff --git a/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/xunit-vstest/CalculatorTests.cs b/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/xunit-vstest/CalculatorTests.cs new file mode 100644 index 0000000000..f735cde2ff --- /dev/null +++ b/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/xunit-vstest/CalculatorTests.cs @@ -0,0 +1,26 @@ +using Xunit; + +namespace MyApp.Tests; + +public class CalculatorTests +{ + [Fact] + public void Add_ReturnsCorrectSum() + { + Assert.Equal(4, 2 + 2); + } + + [Fact] + public void Subtract_ReturnsCorrectDifference() + { + Assert.Equal(0, 2 - 2); + } + + [Theory] + [InlineData(2, 3, 6)] + [InlineData(4, 5, 20)] + public void Multiply_ReturnsCorrectProduct(int a, int b, int expected) + { + Assert.Equal(expected, a * b); + } +} diff --git a/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/xunit-vstest/TestProject.csproj b/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/xunit-vstest/TestProject.csproj new file mode 100644 index 0000000000..3c97eca463 --- /dev/null +++ b/tests/dotnet-test/migrate-vstest-to-mtp/fixtures/xunit-vstest/TestProject.csproj @@ -0,0 +1,13 @@ + + + net8.0 + enable + enable + false + + + + + + +