Skip to content

Commit 7d91314

Browse files
nohwndnohwnd
authored
Add Azure DevOps extension to report errors (#5260)
1 parent a869263 commit 7d91314

34 files changed

+1073
-60
lines changed

‎TestFx.sln

+7
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MSTest.Engine.UnitTests", "
222222
EndProject
223223
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MSTest.SourceGeneration.UnitTests", "test\UnitTests\MSTest.SourceGeneration.UnitTests\MSTest.SourceGeneration.UnitTests.csproj", "{E6C0466E-BE8D-C04F-149A-FD98438F1413}"
224224
EndProject
225+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Testing.Extensions.AzureDevOps", "src\Platform\Microsoft.Testing.Extensions.AzureDevOps\Microsoft.Testing.Extensions.AzureDevOps.csproj", "{F608D3A3-125B-CD88-1D51-8714ED142029}"
226+
EndProject
225227
Global
226228
GlobalSection(SolutionConfigurationPlatforms) = preSolution
227229
Debug|Any CPU = Debug|Any CPU
@@ -524,6 +526,10 @@ Global
524526
{E6C0466E-BE8D-C04F-149A-FD98438F1413}.Debug|Any CPU.Build.0 = Debug|Any CPU
525527
{E6C0466E-BE8D-C04F-149A-FD98438F1413}.Release|Any CPU.ActiveCfg = Release|Any CPU
526528
{E6C0466E-BE8D-C04F-149A-FD98438F1413}.Release|Any CPU.Build.0 = Release|Any CPU
529+
{F608D3A3-125B-CD88-1D51-8714ED142029}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
530+
{F608D3A3-125B-CD88-1D51-8714ED142029}.Debug|Any CPU.Build.0 = Debug|Any CPU
531+
{F608D3A3-125B-CD88-1D51-8714ED142029}.Release|Any CPU.ActiveCfg = Release|Any CPU
532+
{F608D3A3-125B-CD88-1D51-8714ED142029}.Release|Any CPU.Build.0 = Release|Any CPU
527533
EndGlobalSection
528534
GlobalSection(SolutionProperties) = preSolution
529535
HideSolutionNode = FALSE
@@ -615,6 +621,7 @@ Global
615621
{7BA0E74E-798E-4399-2EDE-A23BD5DA78CA} = {E7F15C9C-3928-47AD-8462-64FD29FFCA54}
616622
{2C0DFAC0-5D58-D172-ECE4-CBB78AD03435} = {BB874DF1-44FE-415A-B634-A6B829107890}
617623
{E6C0466E-BE8D-C04F-149A-FD98438F1413} = {BB874DF1-44FE-415A-B634-A6B829107890}
624+
{F608D3A3-125B-CD88-1D51-8714ED142029} = {6AEE1440-FDF0-4729-8196-B24D0E333550}
618625
EndGlobalSection
619626
GlobalSection(ExtensibilityGlobals) = postSolution
620627
SolutionGuid = {31E0F4D5-975A-41CC-933E-545B2201FAF9}

‎eng/Versions.props

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
<Project>
22
<PropertyGroup Label="Version settings">
33
<!-- MSTest version -->
4-
<VersionPrefix>3.9.0</VersionPrefix>
4+
<VersionPrefix>3.10.0</VersionPrefix>
55
<!-- Testing Platform version -->
6-
<TestingPlatformVersionPrefix>1.7.0</TestingPlatformVersionPrefix>
6+
<TestingPlatformVersionPrefix>1.8.0</TestingPlatformVersionPrefix>
77
<PreReleaseVersionLabel>preview</PreReleaseVersionLabel>
88
</PropertyGroup>
99
<PropertyGroup Label="MSTest prod dependencies - darc updated">

‎src/Platform/Microsoft.Testing.Extensions.AzureDevOps/AzDoEscaper.cs

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
3+
4+
using Microsoft.Testing.Platform;
5+
6+
namespace Microsoft.Testing.Extensions.Reporting;
7+
8+
internal static class AzDoEscaper
9+
{
10+
public static string Escape(string value)
11+
{
12+
if (RoslynString.IsNullOrEmpty(value))
13+
{
14+
return value;
15+
}
16+
17+
var result = new StringBuilder(value.Length);
18+
foreach (char c in value)
19+
{
20+
switch (c)
21+
{
22+
case ';':
23+
result.Append("%3B");
24+
break;
25+
case '\r':
26+
result.Append("%0D");
27+
break;
28+
case '\n':
29+
result.Append("%0A");
30+
break;
31+
default:
32+
result.Append(c);
33+
break;
34+
}
35+
}
36+
37+
return result.ToString();
38+
}
39+
}

‎src/Platform/Microsoft.Testing.Extensions.AzureDevOps/AzureDevOpsCommandLineOptions.cs

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
3+
4+
namespace Microsoft.Testing.Extensions.Reporting;
5+
6+
internal static class AzureDevOpsCommandLineOptions
7+
{
8+
public const string AzureDevOpsOptionName = "report-azdo";
9+
public const string AzureDevOpsReportSeverity = "report-azdo-severity";
10+
}

‎src/Platform/Microsoft.Testing.Extensions.AzureDevOps/AzureDevOpsCommandLineProvider.cs

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
3+
4+
using Microsoft.Testing.Extensions.AzureDevOps.Resources;
5+
using Microsoft.Testing.Platform.CommandLine;
6+
using Microsoft.Testing.Platform.Extensions;
7+
using Microsoft.Testing.Platform.Extensions.CommandLine;
8+
using Microsoft.Testing.Platform.Helpers;
9+
10+
namespace Microsoft.Testing.Extensions.Reporting;
11+
12+
internal sealed class AzureDevOpsCommandLineProvider : ICommandLineOptionsProvider
13+
{
14+
private static readonly string[] SeverityOptions = ["error", "warning"];
15+
16+
public string Uid => nameof(AzureDevOpsCommandLineProvider);
17+
18+
public string Version => AppVersion.DefaultSemVer;
19+
20+
public string DisplayName => AzureDevOpsResources.DisplayName;
21+
22+
public string Description => AzureDevOpsResources.Description;
23+
24+
public Task<bool> IsEnabledAsync() => Task.FromResult(true);
25+
26+
public IReadOnlyCollection<CommandLineOption> GetCommandLineOptions()
27+
=>
28+
[
29+
new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsOptionName, AzureDevOpsResources.OptionDescription, ArgumentArity.Zero, false),
30+
new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity, AzureDevOpsResources.SeverityOptionDescription, ArgumentArity.ExactlyOne, false),
31+
];
32+
33+
public Task<ValidationResult> ValidateOptionArgumentsAsync(CommandLineOption commandOption, string[] arguments)
34+
{
35+
if (commandOption.Name == AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity)
36+
{
37+
if (!SeverityOptions.Contains(arguments[0], StringComparer.OrdinalIgnoreCase))
38+
{
39+
return ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.InvalidSeverity, arguments[0]));
40+
}
41+
}
42+
43+
return ValidationResult.ValidTask;
44+
}
45+
46+
public Task<ValidationResult> ValidateCommandLineOptionsAsync(ICommandLineOptions commandLineOptions)
47+
=> ValidationResult.ValidTask;
48+
}

‎src/Platform/Microsoft.Testing.Extensions.AzureDevOps/AzureDevOpsExtensions.cs

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
3+
4+
using Microsoft.Testing.Extensions.Reporting;
5+
using Microsoft.Testing.Extensions.TrxReport.Abstractions;
6+
using Microsoft.Testing.Platform.Builder;
7+
using Microsoft.Testing.Platform.Extensions;
8+
using Microsoft.Testing.Platform.Services;
9+
10+
namespace Microsoft.Testing.Extensions;
11+
12+
/// <summary>
13+
/// Provides extension methods for adding Azure DevOps reporting support to the test application builder.
14+
/// </summary>
15+
public static class AzureDevOpsExtensions
16+
{
17+
/// <summary>
18+
/// Adds support to the test application builder.
19+
/// </summary>
20+
/// <param name="builder">The test application builder.</param>
21+
public static void AddAzureDevOpsProvider(this ITestApplicationBuilder builder)
22+
{
23+
var compositeTestSessionAzDoService =
24+
new CompositeExtensionFactory<AzureDevOpsReporter>(serviceProvider =>
25+
new AzureDevOpsReporter(
26+
serviceProvider.GetCommandLineOptions(),
27+
serviceProvider.GetEnvironment(),
28+
serviceProvider.GetOutputDevice()));
29+
30+
builder.TestHost.AddDataConsumer(compositeTestSessionAzDoService);
31+
32+
builder.CommandLine.AddProvider(() => new AzureDevOpsCommandLineProvider());
33+
}
34+
}

‎src/Platform/Microsoft.Testing.Extensions.AzureDevOps/AzureDevOpsReporter.cs

+167
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
3+
4+
using Microsoft.Testing.Extensions.AzureDevOps;
5+
using Microsoft.Testing.Extensions.AzureDevOps.Resources;
6+
using Microsoft.Testing.Extensions.Reporting;
7+
using Microsoft.Testing.Platform;
8+
using Microsoft.Testing.Platform.CommandLine;
9+
using Microsoft.Testing.Platform.Extensions.Messages;
10+
using Microsoft.Testing.Platform.Extensions.OutputDevice;
11+
using Microsoft.Testing.Platform.Extensions.TestHost;
12+
using Microsoft.Testing.Platform.Helpers;
13+
using Microsoft.Testing.Platform.OutputDevice;
14+
15+
namespace Microsoft.Testing.Extensions.TrxReport.Abstractions;
16+
17+
internal sealed class AzureDevOpsReporter :
18+
IDataConsumer,
19+
IDataProducer,
20+
IOutputDeviceDataProducer
21+
{
22+
private readonly IOutputDevice _outputDisplay;
23+
24+
private static readonly char[] NewlineCharacters = new char[] { '\r', '\n' };
25+
private readonly ICommandLineOptions _commandLine;
26+
private readonly IEnvironment _environment;
27+
private string _severity = "error";
28+
29+
public AzureDevOpsReporter(
30+
ICommandLineOptions commandLine,
31+
IEnvironment environment,
32+
IOutputDevice outputDisplay)
33+
{
34+
_commandLine = commandLine;
35+
_environment = environment;
36+
_outputDisplay = outputDisplay;
37+
}
38+
39+
public Type[] DataTypesConsumed { get; } =
40+
[
41+
typeof(TestNodeUpdateMessage)
42+
];
43+
44+
public Type[] DataTypesProduced { get; } = [typeof(SessionFileArtifact)];
45+
46+
/// <inheritdoc />
47+
public string Uid { get; } = nameof(AzureDevOpsReporter);
48+
49+
/// <inheritdoc />
50+
public string Version { get; } = AppVersion.DefaultSemVer;
51+
52+
/// <inheritdoc />
53+
public string DisplayName { get; } = AzureDevOpsResources.DisplayName;
54+
55+
/// <inheritdoc />
56+
public string Description { get; } = AzureDevOpsResources.Description;
57+
58+
/// <inheritdoc />
59+
public Task<bool> IsEnabledAsync()
60+
{
61+
bool isEnabled = _commandLine.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsOptionName)
62+
&& string.Equals(_environment.GetEnvironmentVariable("TF_BUILD"), "true", StringComparison.OrdinalIgnoreCase);
63+
64+
if (isEnabled)
65+
{
66+
bool found = _commandLine.TryGetOptionArgumentList(AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity, out string[]? arguments);
67+
if (found && arguments?.Length > 0)
68+
{
69+
_severity = arguments[0].ToLowerInvariant();
70+
}
71+
}
72+
73+
return Task.FromResult(isEnabled);
74+
}
75+
76+
public async Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationToken cancellationToken)
77+
{
78+
if (cancellationToken.IsCancellationRequested)
79+
{
80+
return;
81+
}
82+
83+
if (value is not TestNodeUpdateMessage nodeUpdateMessage)
84+
{
85+
return;
86+
}
87+
88+
TestNodeStateProperty nodeState = nodeUpdateMessage.TestNode.Properties.Single<TestNodeStateProperty>();
89+
90+
switch (nodeState)
91+
{
92+
case FailedTestNodeStateProperty failed:
93+
await WriteExceptionAsync(failed.Explanation, failed.Exception);
94+
break;
95+
case ErrorTestNodeStateProperty error:
96+
await WriteExceptionAsync(error.Explanation, error.Exception);
97+
break;
98+
case CancelledTestNodeStateProperty cancelled:
99+
await WriteExceptionAsync(cancelled.Explanation, cancelled.Exception);
100+
break;
101+
case TimeoutTestNodeStateProperty timeout:
102+
await WriteExceptionAsync(timeout.Explanation, timeout.Exception);
103+
break;
104+
}
105+
106+
return;
107+
}
108+
109+
private async Task WriteExceptionAsync(string? explanation, Exception? exception)
110+
{
111+
if (exception == null || exception.StackTrace == null)
112+
{
113+
return;
114+
}
115+
116+
string message = explanation ?? exception.Message;
117+
118+
if (message == null)
119+
{
120+
return;
121+
}
122+
123+
string stackTrace = exception.StackTrace;
124+
int index = stackTrace.IndexOfAny(NewlineCharacters);
125+
string firstLine = index == -1 ? stackTrace : stackTrace.Substring(0, index);
126+
if (firstLine != null)
127+
{
128+
(string Code, string File, int LineNumber)? location = GetStackFrameLocation(firstLine);
129+
if (location != null)
130+
{
131+
string root = RootFinder.Find();
132+
string file = location.Value.File;
133+
string relativePath = file.StartsWith(root, StringComparison.CurrentCultureIgnoreCase) ? file.Substring(root.Length) : file;
134+
string relativeNormalizedPath = relativePath.Replace('\\', '/');
135+
136+
string err = AzDoEscaper.Escape(message);
137+
138+
string line = $"##vso[task.logissue type={_severity};sourcepath={relativeNormalizedPath};linenumber={location.Value.LineNumber};columnnumber=1]{err}";
139+
await _outputDisplay.DisplayAsync(this, new FormattedTextOutputDeviceData(line));
140+
}
141+
}
142+
}
143+
144+
internal /* for testing */ static (string Code, string File, int LineNumber)? GetStackFrameLocation(string stackTraceLine)
145+
{
146+
Match match = StackTraceHelper.GetFrameRegex().Match(stackTraceLine);
147+
if (!match.Success)
148+
{
149+
return null;
150+
}
151+
152+
bool weHaveFilePathAndCodeLine = !RoslynString.IsNullOrWhiteSpace(match.Groups["code"].Value);
153+
if (!weHaveFilePathAndCodeLine)
154+
{
155+
return null;
156+
}
157+
158+
if (RoslynString.IsNullOrWhiteSpace(match.Groups["file"].Value))
159+
{
160+
return null;
161+
}
162+
163+
int line = int.TryParse(match.Groups["line"].Value, out int value) ? value : 0;
164+
165+
return (match.Groups["code"].Value, match.Groups["file"].Value, line);
166+
}
167+
}

‎src/Platform/Microsoft.Testing.Extensions.AzureDevOps/BannedSymbols.txt

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
T:System.ArgumentNullException; Use 'ArgumentGuard' instead
2+
P:System.DateTime.Now; Use 'IClock' instead
3+
P:System.DateTime.UtcNow; Use 'IClock' instead
4+
M:System.Threading.Tasks.Task.Run(System.Action); Use 'ITask' instead
5+
M:System.Threading.Tasks.Task.WhenAll(System.Threading.Tasks.Task[]); Use 'ITask' instead
6+
M:System.Threading.Tasks.Task.WhenAll(System.Collections.Generic.IEnumerable{System.Threading.Tasks.Task}); Use 'ITask' instead
7+
M:System.String.IsNullOrEmpty(System.String); Use 'RoslynString.IsNullOrEmpty' instead
8+
M:System.String.IsNullOrWhiteSpace(System.String); Use 'RoslynString.IsNullOrWhiteSpace' instead
9+
M:System.Diagnostics.Debug.Assert(System.Boolean); Use 'RoslynDebug.Assert' instead
10+
M:System.Diagnostics.Debug.Assert(System.Boolean,System.String); Use 'RoslynDebug.Assert' instead

0 commit comments

Comments
 (0)