Add status checks to the test runner

Contributes to dotnet#1125 Replace the existing logging code with the utility class that writes to the console, and adds a status check record. Log failures as status checks, but status is only to the output console.
BillWagner · Sep 19, 2024 · d6bf6dc · d6bf6dc
1 parent 874bfc9
commit d6bf6dc
Show file tree

Hide file tree

Showing 12 changed files with 78 additions and 34 deletions.
diff --git a/.github/workflows/renumber-sections.yaml b/.github/workflows/renumber-sections.yaml
@@ -16,6 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       checks: write
+      pull-requests: write
     env:
       DOTNET_NOLOGO: true
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

diff --git a/.github/workflows/test-examples.yaml b/.github/workflows/test-examples.yaml
@@ -16,8 +16,13 @@ on:
 jobs:
   test-extraction-and-runner:
     runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      pull-requests: write
     env:
       DOTNET_NOLOGO: true
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      HEAD_SHA: ${{ github.event.pull_request.head.sha }}
 
     steps:
     - name: Check out our repo

diff --git a/.github/workflows/word-converter.yaml b/.github/workflows/word-converter.yaml
@@ -16,6 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       checks: write
+      pull-requests: write
     env:
       DOTNET_NOLOGO: true
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

diff --git a/.gitignore b/.gitignore
@@ -360,3 +360,6 @@ test-grammar/
 
 # don't checkin jar files:
 *.jar
+
+# don't checkin launchSettings:
+**/launchSettings.json
diff --git a/tools/ExampleTester/ExampleTester.csproj b/tools/ExampleTester/ExampleTester.csproj
@@ -16,6 +16,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\ExampleExtractor\ExampleExtractor.csproj" />
+    <ProjectReference Include="..\Utilities\Utilities.csproj" />
   </ItemGroup>
 
 </Project>
diff --git a/tools/ExampleTester/GeneratedExample.cs b/tools/ExampleTester/GeneratedExample.cs
@@ -5,6 +5,7 @@
 using Newtonsoft.Json;
 using System.Reflection;
 using System.Text;
+using Utilities;
 
 namespace ExampleTester;
 
@@ -33,9 +34,9 @@ private static GeneratedExample Load(string directory)
         return new GeneratedExample(directory);
     }
 
-    internal async Task<bool> Test(TesterConfiguration configuration)
+    internal async Task<bool> Test(TesterConfiguration configuration, StatusCheckLogger logger)
     {
-        var outputLines = new List<string> { $"Testing {Metadata.Name} from {Metadata.Source}" };
+        logger.ConsoleOnlyLog(new StatusCheckMessage(Metadata.Source, Metadata.StartLine, Metadata.EndLine, $"Testing {Metadata.Name} from {Metadata.Source}", "ExampleTester"));
 
         // Explicitly do a release build, to avoid implicitly defining DEBUG.
         var properties = new Dictionary<string, string> { { "Configuration", "Release" } };
@@ -52,21 +53,16 @@ internal async Task<bool> Test(TesterConfiguration configuration)
         }
 
         bool ret = true;
-        ret &= ValidateDiagnostics("errors", DiagnosticSeverity.Error, Metadata.ExpectedErrors);
-        ret &= ValidateDiagnostics("warnings", DiagnosticSeverity.Warning, Metadata.ExpectedWarnings, Metadata.IgnoredWarnings);
+        ret &= ValidateDiagnostics("errors", DiagnosticSeverity.Error, Metadata.ExpectedErrors, logger);
+        ret &= ValidateDiagnostics("warnings", DiagnosticSeverity.Warning, Metadata.ExpectedWarnings, logger, Metadata.IgnoredWarnings);
         // Don't try to validate output if we've already failed in terms of errors and warnings, or if we expect errors.
         if (ret && Metadata.ExpectedErrors is null)
         {
             ret &= ValidateOutput();
         }
-
-        if (!ret || !configuration.Quiet)
-        {
-            outputLines.ForEach(Console.WriteLine);
-        }
         return ret;
 
-        bool ValidateDiagnostics(string type, DiagnosticSeverity severity, List<string> expected, List<string>? ignored = null)
+        bool ValidateDiagnostics(string type, DiagnosticSeverity severity, List<string> expected, StatusCheckLogger logger, List<string>? ignored = null)
         {
             expected ??= new List<string>();
             ignored ??= new List<string>();
@@ -81,10 +77,12 @@ bool ValidateDiagnostics(string type, DiagnosticSeverity severity, List<string>
             bool ret = ValidateExpectedAgainstActual(type, expected, actualIds);
             if (!ret)
             {
-                outputLines.Add($"  Details of actual {type}:");
+                logger.LogFailure(new StatusCheckMessage(Metadata.Source, Metadata.StartLine, Metadata.EndLine, $"  Details of actual {type}:", "ExampleTester"));
                 foreach (var diagnostic in actualDiagnostics)
                 {
-                    outputLines.Add($"    Line {diagnostic.Location.GetLineSpan().StartLinePosition.Line + 1}: {diagnostic.Id}: {diagnostic.GetMessage()}");
+                    logger.LogFailure(new StatusCheckMessage(Metadata.Source, Metadata.StartLine, Metadata.EndLine,
+                        $"    Line {diagnostic.Location.GetLineSpan().StartLinePosition.Line + 1}: {diagnostic.Id}: {diagnostic.GetMessage()}",
+                        "ExampleTester"));
                 }
             }
             return ret;
@@ -97,7 +95,7 @@ bool ValidateOutput()
             {
                 if (Metadata.ExpectedOutput != null)
                 {
-                    outputLines.Add("  Output expected, but project has no entry point.");
+                    logger.LogFailure(new StatusCheckMessage(Metadata.Source, Metadata.StartLine, Metadata.EndLine, "  Output expected, but project has no entry point.", "ExampleTester"));
                     return false;
                 }
                 return true;
@@ -114,21 +112,21 @@ bool ValidateOutput()
             var emitResult = compilation.Emit(ms);
             if (!emitResult.Success)
             {
-                outputLines.Add("  Failed to emit assembly");
+                logger.LogFailure(new StatusCheckMessage(Metadata.Source, Metadata.StartLine, Metadata.EndLine, "  Failed to emit assembly", "ExampleTester"));
                 return false;
             }
 
             var generatedAssembly = Assembly.Load(ms.ToArray());
             var type = generatedAssembly.GetType(typeName);
             if (type is null)
             {
-                outputLines.Add($"  Failed to find entry point type {typeName}");
+                logger.LogFailure(new StatusCheckMessage(Metadata.Source, Metadata.StartLine, Metadata.EndLine, $"  Failed to find entry point type {typeName}", "ExampleTester"));
                 return false;
             }
             var method = type.GetMethod(methodName, BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static);
             if (method is null)
             {
-                outputLines.Add($"  Failed to find entry point method {typeName}.{methodName}");
+                logger.LogFailure(new StatusCheckMessage(Metadata.Source, Metadata.StartLine, Metadata.EndLine, $"  Failed to find entry point method {typeName}.{methodName}", "ExampleTester"));
                 return false;
             }
             var arguments = method.GetParameters().Any()
@@ -197,7 +195,7 @@ bool MaybeReportError(bool result, string message)
             {
                 if (!result)
                 {
-                    outputLines.Add(message);
+                    logger.LogFailure(new StatusCheckMessage(Metadata.Source, Metadata.StartLine, Metadata.EndLine, message, "ExampleTester"));
                 }
                 return result;
             }
@@ -207,7 +205,8 @@ bool ValidateExpectedAgainstActual(string type, List<string> expected, List<stri
         {
             if (!expected.SequenceEqual(actual))
             {
-                outputLines.Add($"  Mismatched {type}: Expected {string.Join(", ", expected)}; Was {string.Join(", ", actual)}");
+                logger.LogFailure(new StatusCheckMessage(Metadata.Source, Metadata.StartLine, Metadata.EndLine,
+                    $"  Mismatched {type}: Expected {string.Join(", ", expected)}; Was {string.Join(", ", actual)}", "ExampleTester"));
                 return false;
             }
             return true;

diff --git a/tools/ExampleTester/Program.cs b/tools/ExampleTester/Program.cs
@@ -1,9 +1,20 @@
 using ExampleTester;
 using System.CommandLine;
+using Utilities;
+
+var logger = new StatusCheckLogger("..", "Example tester");
+var headSha = Environment.GetEnvironmentVariable("HEAD_SHA");
+var token = Environment.GetEnvironmentVariable("GH_TOKEN");
 
 var rootCommand = new RootCommand();
 new TesterConfigurationBinder().ConfigureCommand(rootCommand, ExecuteAsync);
-return rootCommand.Invoke(args);
+int exitCode = rootCommand.Invoke(args);
+
+if ((token is not null) && (headSha is not null))
+{
+    await logger.BuildCheckRunResult(token, "dotnet", "csharpstandard", headSha);
+}
+return exitCode;
 
 async Task<int> ExecuteAsync(TesterConfiguration configuration)
 {
@@ -31,7 +42,7 @@ async Task<int> ExecuteAsync(TesterConfiguration configuration)
     foreach (var example in examples)
     {
         // The Run method explains any failures, we just need to count them.
-        if (!await example.Test(configuration))
+        if (!await example.Test(configuration, logger))
         {
             failures++;
         }
@@ -42,4 +53,4 @@ async Task<int> ExecuteAsync(TesterConfiguration configuration)
     Console.WriteLine($"Failures: {failures}");
 
     return failures;
-}
+}
diff --git a/tools/MarkdownConverter/Spec/Reporter.cs b/tools/MarkdownConverter/Spec/Reporter.cs
@@ -59,14 +59,14 @@ public void Error(string code, string msg, SourceLocation? loc = null)
     {
         loc = loc ?? Location;
         IncrementErrors();
-        githubLogger.LogFailure(new Diagnostic(loc.File ?? "mdspec2docx", loc.StartLine, loc.EndLine, msg, code));
+        githubLogger.LogFailure(new StatusCheckMessage(loc.File ?? "mdspec2docx", loc.StartLine, loc.EndLine, msg, code));
     }
 
     public void Warning(string code, string msg, SourceLocation? loc = null, int lineOffset = 0)
     {
         loc = loc ?? Location;
         IncrementWarnings();
-        githubLogger.LogWarning(new Diagnostic(loc.File ?? "mdspec2docx", loc.StartLine+lineOffset, loc.EndLine+lineOffset, msg, code));
+        githubLogger.LogWarning(new StatusCheckMessage(loc.File ?? "mdspec2docx", loc.StartLine+lineOffset, loc.EndLine+lineOffset, msg, code));
     }
 
     public void Log(string code, string msg, SourceLocation? loc = null)

diff --git a/tools/StandardAnchorTags/ReferenceUpdateProcessor.cs b/tools/StandardAnchorTags/ReferenceUpdateProcessor.cs
@@ -62,7 +62,7 @@ private string ProcessSectionLinks(string line, int lineNumber, string path)
             if ((referenceText.Length > 1) &&
                 (!linkMap.ContainsKey(referenceText)))
             {
-                var diagnostic = new Diagnostic(path, lineNumber, lineNumber, $"`{referenceText}` not found", DiagnosticIDs.TOC002);
+                var diagnostic = new StatusCheckMessage(path, lineNumber, lineNumber, $"`{referenceText}` not found", DiagnosticIDs.TOC002);
                 logger.LogFailure(diagnostic);
             } else
             {

diff --git a/tools/StandardAnchorTags/TocSectionNumberBuilder.cs b/tools/StandardAnchorTags/TocSectionNumberBuilder.cs
@@ -69,7 +69,7 @@ public async Task AddFrontMatterTocEntries(string fileName)
             return;
         }
         // Getting here means this file doesn't have an H1. That's an error:
-        var diagnostic = new Diagnostic(path, 1, 1, "File doesn't have an H1 tag as its first line.", DiagnosticIDs.TOC001);
+        var diagnostic = new StatusCheckMessage(path, 1, 1, "File doesn't have an H1 tag as its first line.", DiagnosticIDs.TOC001);
         logger.LogFailure(diagnostic);
     }
 

diff --git a/tools/Utilities/StatusCheckLogger.cs b/tools/Utilities/StatusCheckLogger.cs
@@ -10,7 +10,7 @@ namespace Utilities;
 /// <param name="Id">The error message ID</param>
 /// <param name="StartLine">The start line (index from 1)</param>
 /// <param name="EndLine">The end line (index from 1)</param>
-public record Diagnostic(string file, int StartLine, int EndLine, string Message, string Id);
+public record StatusCheckMessage(string file, int StartLine, int EndLine, string Message, string Id);
 
 /// <summary>
 /// This class writes the status of the check to the console in the format GitHub supports
@@ -30,7 +30,25 @@ public class StatusCheckLogger(string pathToRoot, string toolName)
     // Utility method to format the path to unix style, from the root of the repository.
     private string FormatPath(string path) => Path.GetRelativePath(pathToRoot, path).Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
 
-    private void WriteMessageToConsole(string prefix, Diagnostic d) => Console.WriteLine($"{prefix}{toolName}-{d.Id}::file={FormatPath(d.file)},line={d.StartLine}::{d.Message}");
+    private void WriteMessageToConsole(string prefix, StatusCheckMessage d) => Console.WriteLine($"{prefix}{toolName}-{d.Id}::file={FormatPath(d.file)},line={d.StartLine}::{d.Message}");
+
+    /// <summary>
+    /// Log a notice from the status check to the console only
+    /// </summary>
+    /// <param name="d">The diagnostic</param>
+    /// <remarks>
+    /// Log the diagnostic information to console. These will only appear in the console window, not
+    /// as annotations on the changes in the PR. 
+    /// </remarks>
+    public void ConsoleOnlyLog(StatusCheckMessage d)
+    {
+        WriteMessageToConsole("", d);
+        annotations.Add(
+            new(FormatPath(d.file),
+            d.StartLine, d.EndLine,
+            CheckAnnotationLevel.Notice, $"{d.Id}::{d.Message}")
+        );
+    }
 
     /// <summary>
     /// Log a notice from the status check
@@ -40,7 +58,7 @@ public class StatusCheckLogger(string pathToRoot, string toolName)
     /// Add the diagnostic to the annotation list and
     /// log the diagnostic information to console.
     /// </remarks>
-    public void LogNotice(Diagnostic d)
+    public void LogNotice(StatusCheckMessage d)
     {
         WriteMessageToConsole("", d);
         annotations.Add(
@@ -59,7 +77,7 @@ public void LogNotice(Diagnostic d)
     /// log the warning notice to the console.
     /// Warnings are logged, but the process reports "success" to GitHub.
     /// </remarks>
-    public void LogWarning(Diagnostic d)
+    public void LogWarning(StatusCheckMessage d)
     {
         WriteMessageToConsole("⚠️", d);
         annotations.Add(
@@ -76,11 +94,11 @@ public void LogWarning(Diagnostic d)
     /// <remarks>
     /// Add the diagnostic to the annotation list and
     /// log the failure notice to the console.
-    /// This method is distinct from <see cref="ExitOnFailure(Diagnostic)"/> in
+    /// This method is distinct from <see cref="ExitOnFailure(StatusCheckMessage)"/> in
     /// that this method does not throw an exception. Its purpose is to log
     /// the failure but allow the tool to continue running further checks.
     /// </remarks>
-    public void LogFailure(Diagnostic d)
+    public void LogFailure(StatusCheckMessage d)
     {
         WriteMessageToConsole("❌", d);
         annotations.Add(
@@ -98,11 +116,11 @@ public void LogFailure(Diagnostic d)
     /// <remarks>
     /// Add the diagnostic to the annotation list and
     /// log the failure notice to the console.
-    /// This method is distinct from <see cref="LogFailure(Diagnostic)"/> in
+    /// This method is distinct from <see cref="LogFailure(StatusCheckMessage)"/> in
     /// that this method throws an exception. Its purpose is to log
     /// the failure and immediately exit, foregoing any further checks.
     /// </remarks>
-    public void ExitOnFailure(Diagnostic d)
+    public void ExitOnFailure(StatusCheckMessage d)
     {
         LogFailure(d);
         throw new InvalidOperationException(d.Message);
@@ -138,9 +156,11 @@ public async Task BuildCheckRunResult(string token, string owner, string repo, s
         }
         // If the token does not have the correct permissions, we will get a 403
         // Once running on a branch on the dotnet org, this should work correctly.
-        catch (ForbiddenException)
+        catch (ForbiddenException e)
         {
             Console.WriteLine("===== WARNING: Could not create a check run.=====");
+            Console.WriteLine("Exception details:");
+            Console.WriteLine(e);
         }
     }
 }
diff --git a/tools/tools.sln b/tools/tools.sln
@@ -14,6 +14,9 @@ EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ExampleExtractor", "ExampleExtractor\ExampleExtractor.csproj", "{571E69B9-07A3-4682-B692-876B016315CD}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ExampleTester", "ExampleTester\ExampleTester.csproj", "{829FE7D6-B7E7-48DF-923A-73A79921E997}"
+	ProjectSection(ProjectDependencies) = postProject
+		{835C6333-BDB5-4DEC-B3BE-4300E3F948AF} = {835C6333-BDB5-4DEC-B3BE-4300E3F948AF}
+	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ExampleFormatter", "ExampleFormatter\ExampleFormatter.csproj", "{82D1A159-5637-48C4-845D-CC1390995CC2}"
 EndProject