From b5e1610f67c8f4d3afcdb7e4fafafe490d8e001d Mon Sep 17 00:00:00 2001 From: "gel@microsoft.com" Date: Wed, 1 Mar 2023 14:12:53 -0800 Subject: [PATCH 1/2] Add option to ignore missing packages when publishing Some time we need to publish compiler packages before release versions of VS packages are available publicly. This would make it easier to publish them separately --- .../Commands/NuGetPublishCommand.cs | 5 +- src/dotnet-roslyn-tools/NuGet/Helpers.cs | 107 ++++++++++++++ src/dotnet-roslyn-tools/NuGet/NuGetPrepare.cs | 45 +----- src/dotnet-roslyn-tools/NuGet/NuGetPublish.cs | 130 +++++------------- 4 files changed, 148 insertions(+), 139 deletions(-) create mode 100644 src/dotnet-roslyn-tools/NuGet/Helpers.cs diff --git a/src/dotnet-roslyn-tools/Commands/NuGetPublishCommand.cs b/src/dotnet-roslyn-tools/Commands/NuGetPublishCommand.cs index 98d1acff3a..989f7a731f 100644 --- a/src/dotnet-roslyn-tools/Commands/NuGetPublishCommand.cs +++ b/src/dotnet-roslyn-tools/Commands/NuGetPublishCommand.cs @@ -21,6 +21,7 @@ internal class NuGetPublishCommand internal static readonly Option SourceOption = new Option(new[] { "--source", "-s" }, () => "https://www.nuget.org", "Package source (URL, UNC/folder path or package source name) to use."); internal static readonly Option ApiKeyOption = new Option(new[] { "--api-key", "-k" }, "The API key for the server.") { IsRequired = true }; internal static readonly Option UnlistedOption = new Option(new[] { "--unlisted", "-u" }, "Whether to publish the packages as unlisted."); + internal static readonly Option IgnoreMissingPackagesOption = new Option(new[] { "--ignore", "-i" }, "Whether to ignore missing packages."); public static Symbol GetCommand() { @@ -30,6 +31,7 @@ public static Symbol GetCommand() SourceOption, ApiKeyOption, UnlistedOption, + IgnoreMissingPackagesOption, VerbosityOption }; command.Handler = s_nuGetPublishCommandHandler; @@ -47,8 +49,9 @@ public async Task InvokeAsync(InvocationContext context) var source = context.ParseResult.GetValueForOption(SourceOption)!; var apiKey = context.ParseResult.GetValueForOption(ApiKeyOption)!; var unlisted = context.ParseResult.GetValueForOption(UnlistedOption)!; + var ignore = context.ParseResult.GetValueForOption(IgnoreMissingPackagesOption)!; - return await NuGetPublish.PublishAsync(repoName, source, apiKey, unlisted, logger); + return await NuGetPublish.PublishAsync(repoName, source, apiKey, unlisted, ignore, logger); } } } diff --git a/src/dotnet-roslyn-tools/NuGet/Helpers.cs b/src/dotnet-roslyn-tools/NuGet/Helpers.cs new file mode 100644 index 0000000000..ab65ed4222 --- /dev/null +++ b/src/dotnet-roslyn-tools/NuGet/Helpers.cs @@ -0,0 +1,107 @@ +// Licensed to the.NET Foundation under one or more agreements. +// The.NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.RoslynTools.NuGet +{ + internal static class Helpers + { + public static readonly string[] RoslynPackageIds = + { + "Microsoft.CodeAnalysis", + "Microsoft.CodeAnalysis.Common", + "Microsoft.CodeAnalysis.Compilers", + "Microsoft.CodeAnalysis.CSharp", + "Microsoft.CodeAnalysis.CSharp.CodeStyle", + "Microsoft.CodeAnalysis.CSharp.Features", + "Microsoft.CodeAnalysis.CSharp.Scripting", + "Microsoft.CodeAnalysis.CSharp.Workspaces", + "Microsoft.CodeAnalysis.EditorFeatures.Text", + "Microsoft.CodeAnalysis.Features", + "Microsoft.CodeAnalysis.Scripting", + "Microsoft.CodeAnalysis.Scripting.Common", + "Microsoft.CodeAnalysis.VisualBasic", + "Microsoft.CodeAnalysis.VisualBasic.CodeStyle", + "Microsoft.CodeAnalysis.VisualBasic.Features", + "Microsoft.CodeAnalysis.VisualBasic.Workspaces", + "Microsoft.CodeAnalysis.Workspaces.Common", + "Microsoft.CodeAnalysis.Workspaces.MSBuild", + "Microsoft.Net.Compilers.Toolset", + "Microsoft.VisualStudio.LanguageServices" + }; + + public static readonly string[] RoslynSdkPackageIds = + { + "Microsoft.CodeAnalysis.Analyzer.Testing", + "Microsoft.CodeAnalysis.CodeFix.Testing", + "Microsoft.CodeAnalysis.CodeRefactoring.Testing", + "Microsoft.CodeAnalysis.CSharp.Analyzer.Testing", + "Microsoft.CodeAnalysis.CSharp.Analyzer.Testing.MSTest", + "Microsoft.CodeAnalysis.CSharp.Analyzer.Testing.NUnit", + "Microsoft.CodeAnalysis.CSharp.Analyzer.Testing.XUnit", + "Microsoft.CodeAnalysis.CSharp.CodeFix.Testing", + "Microsoft.CodeAnalysis.CSharp.CodeFix.Testing.MSTest", + "Microsoft.CodeAnalysis.CSharp.CodeFix.Testing.NUnit", + "Microsoft.CodeAnalysis.CSharp.CodeFix.Testing.XUnit", + "Microsoft.CodeAnalysis.CSharp.CodeRefactoring.Testing", + "Microsoft.CodeAnalysis.CSharp.CodeRefactoring.Testing.MSTest", + "Microsoft.CodeAnalysis.CSharp.CodeRefactoring.Testing.NUnit", + "Microsoft.CodeAnalysis.CSharp.CodeRefactoring.Testing.XUnit", + "Microsoft.CodeAnalysis.CSharp.SourceGenerators.Testing", + "Microsoft.CodeAnalysis.CSharp.SourceGenerators.Testing.MSTest", + "Microsoft.CodeAnalysis.CSharp.SourceGenerators.Testing.NUnit", + "Microsoft.CodeAnalysis.CSharp.SourceGenerators.Testing.XUnit", + "Microsoft.CodeAnalysis.SourceGenerators.Testing", + "Microsoft.CodeAnalysis.Testing.Verifiers.MSTest", + "Microsoft.CodeAnalysis.Testing.Verifiers.NUnit", + "Microsoft.CodeAnalysis.Testing.Verifiers.XUnit", + "Microsoft.CodeAnalysis.VisualBasic.Analyzer.Testing", + "Microsoft.CodeAnalysis.VisualBasic.Analyzer.Testing.MSTest", + "Microsoft.CodeAnalysis.VisualBasic.Analyzer.Testing.NUnit", + "Microsoft.CodeAnalysis.VisualBasic.Analyzer.Testing.XUnit", + "Microsoft.CodeAnalysis.VisualBasic.CodeFix.Testing", + "Microsoft.CodeAnalysis.VisualBasic.CodeFix.Testing.MSTest", + "Microsoft.CodeAnalysis.VisualBasic.CodeFix.Testing.NUnit", + "Microsoft.CodeAnalysis.VisualBasic.CodeFix.Testing.XUnit", + "Microsoft.CodeAnalysis.VisualBasic.CodeRefactoring.Testing", + "Microsoft.CodeAnalysis.VisualBasic.CodeRefactoring.Testing.MSTest", + "Microsoft.CodeAnalysis.VisualBasic.CodeRefactoring.Testing.NUnit", + "Microsoft.CodeAnalysis.VisualBasic.CodeRefactoring.Testing.XUnit", + "Microsoft.CodeAnalysis.VisualBasic.SourceGenerators.Testing", + "Microsoft.CodeAnalysis.VisualBasic.SourceGenerators.Testing.MSTest", + "Microsoft.CodeAnalysis.VisualBasic.SourceGenerators.Testing.NUnit", + "Microsoft.CodeAnalysis.VisualBasic.SourceGenerators.Testing.XUnit" + }; + + public static bool TryDetermineRoslynPackageVersion([NotNullWhen(returnValue: true)] out string? version) + { + var packageFileName = Directory.GetFiles(Environment.CurrentDirectory, "Microsoft.Net.Compilers.Toolset.*.nupkg").FirstOrDefault(); + if (packageFileName is null) + { + version = null; + return false; + } + + version = Path.GetFileNameWithoutExtension(packageFileName).Substring(32); + return true; + } + + public static bool TryDetermineRoslynSdkPackageVersion([NotNullWhen(returnValue: true)] out string? version) + { + var packageFileName = Directory.GetFiles(Environment.CurrentDirectory, "Microsoft.CodeAnalysis.Analyzer.Testing.*.nupkg").FirstOrDefault(); + if (packageFileName is null) + { + version = null; + return false; + } + + version = Path.GetFileNameWithoutExtension(packageFileName).Substring(40); + return true; + } + + public static string GetPackageFileName(string packageId, string version) => $"{packageId}.{version}.nupkg"; + + } +} diff --git a/src/dotnet-roslyn-tools/NuGet/NuGetPrepare.cs b/src/dotnet-roslyn-tools/NuGet/NuGetPrepare.cs index 9b0e0737f9..be44e1c8d6 100644 --- a/src/dotnet-roslyn-tools/NuGet/NuGetPrepare.cs +++ b/src/dotnet-roslyn-tools/NuGet/NuGetPrepare.cs @@ -3,7 +3,8 @@ // See the License.txt file in the project root for more information. using Microsoft.Extensions.Logging; -using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Contracts; +using static Microsoft.RoslynTools.NuGet.Helpers; namespace Microsoft.RoslynTools.NuGet { @@ -11,36 +12,11 @@ internal class NuGetPrepare { private const string NotPublishedDirectoryName = "NotPublished"; - private static readonly string[] RoslynPackageIds = - { - "Microsoft.CodeAnalysis", - "Microsoft.CodeAnalysis.Common", - "Microsoft.CodeAnalysis.Compilers", - "Microsoft.CodeAnalysis.CSharp", - "Microsoft.CodeAnalysis.CSharp.CodeStyle", - "Microsoft.CodeAnalysis.CSharp.Features", - "Microsoft.CodeAnalysis.CSharp.Scripting", - "Microsoft.CodeAnalysis.CSharp.Workspaces", - "Microsoft.CodeAnalysis.EditorFeatures.Text", - "Microsoft.CodeAnalysis.Features", - "Microsoft.CodeAnalysis.Scripting", - "Microsoft.CodeAnalysis.Scripting.Common", - "Microsoft.CodeAnalysis.VisualBasic", - "Microsoft.CodeAnalysis.VisualBasic.CodeStyle", - "Microsoft.CodeAnalysis.VisualBasic.Features", - "Microsoft.CodeAnalysis.VisualBasic.Workspaces", - "Microsoft.CodeAnalysis.Workspaces.Common", - "Microsoft.CodeAnalysis.Workspaces.MSBuild", - "Microsoft.Net.Compilers.Toolset", - "Microsoft.VisualStudio.LanguageServices" - }; - internal static async Task PrepareAsync(ILogger logger) { try { string? version; - var determinedVersion = TryDetermineRoslynPackageVersion(out version); if (!determinedVersion) @@ -49,10 +25,12 @@ internal static async Task PrepareAsync(ILogger logger) return 1; } + Contract.Assert(version is not null); + logger.LogInformation($"Moving {version} packages..."); var publishedPackages = RoslynPackageIds - .Select(packageId => $"{packageId}.{version}.nupkg") + .Select(packageId => GetPackageFileName(packageId, version)) .ToHashSet(); if (!Directory.Exists(NotPublishedDirectoryName)) @@ -85,19 +63,6 @@ internal static async Task PrepareAsync(ILogger logger) return 0; - static bool TryDetermineRoslynPackageVersion([NotNullWhen(returnValue: true)] out string? version) - { - var packageFileName = Directory.GetFiles(Environment.CurrentDirectory, "Microsoft.Net.Compilers.Toolset.*.nupkg").FirstOrDefault(); - if (packageFileName is null) - { - version = null; - return false; - } - - version = Path.GetFileNameWithoutExtension(packageFileName).Substring(32); - return true; - } - static bool IsPublishedPackage(string packagePath, HashSet publishedPackages) { var packageFileName = Path.GetFileName(packagePath); diff --git a/src/dotnet-roslyn-tools/NuGet/NuGetPublish.cs b/src/dotnet-roslyn-tools/NuGet/NuGetPublish.cs index 1ceae7967e..e3f91fa1d3 100644 --- a/src/dotnet-roslyn-tools/NuGet/NuGetPublish.cs +++ b/src/dotnet-roslyn-tools/NuGet/NuGetPublish.cs @@ -3,8 +3,10 @@ // See the License.txt file in the project root for more information. using Microsoft.Extensions.Logging; -using System.Diagnostics.CodeAnalysis; using Microsoft.RoslynTools.Utilities; +using System.Diagnostics.Contracts; + +using static Microsoft.RoslynTools.NuGet.Helpers; namespace Microsoft.RoslynTools.NuGet { @@ -13,74 +15,7 @@ internal class NuGetPublish public const string RoslynRepo = "roslyn"; public const string RoslynSdkRepo = "roslyn-sdk"; - private static readonly string[] RoslynPackageIds = - { - "Microsoft.CodeAnalysis", - "Microsoft.CodeAnalysis.Common", - "Microsoft.CodeAnalysis.Compilers", - "Microsoft.CodeAnalysis.CSharp", - "Microsoft.CodeAnalysis.CSharp.CodeStyle", - "Microsoft.CodeAnalysis.CSharp.Features", - "Microsoft.CodeAnalysis.CSharp.Scripting", - "Microsoft.CodeAnalysis.CSharp.Workspaces", - "Microsoft.CodeAnalysis.EditorFeatures.Text", - "Microsoft.CodeAnalysis.Features", - "Microsoft.CodeAnalysis.Scripting", - "Microsoft.CodeAnalysis.Scripting.Common", - "Microsoft.CodeAnalysis.VisualBasic", - "Microsoft.CodeAnalysis.VisualBasic.CodeStyle", - "Microsoft.CodeAnalysis.VisualBasic.Features", - "Microsoft.CodeAnalysis.VisualBasic.Workspaces", - "Microsoft.CodeAnalysis.Workspaces.Common", - "Microsoft.CodeAnalysis.Workspaces.MSBuild", - "Microsoft.Net.Compilers.Toolset", - "Microsoft.VisualStudio.LanguageServices" - }; - - private static readonly string[] RoslynSdkPackageIds = - { - "Microsoft.CodeAnalysis.Analyzer.Testing", - "Microsoft.CodeAnalysis.CodeFix.Testing", - "Microsoft.CodeAnalysis.CodeRefactoring.Testing", - "Microsoft.CodeAnalysis.CSharp.Analyzer.Testing", - "Microsoft.CodeAnalysis.CSharp.Analyzer.Testing.MSTest", - "Microsoft.CodeAnalysis.CSharp.Analyzer.Testing.NUnit", - "Microsoft.CodeAnalysis.CSharp.Analyzer.Testing.XUnit", - "Microsoft.CodeAnalysis.CSharp.CodeFix.Testing", - "Microsoft.CodeAnalysis.CSharp.CodeFix.Testing.MSTest", - "Microsoft.CodeAnalysis.CSharp.CodeFix.Testing.NUnit", - "Microsoft.CodeAnalysis.CSharp.CodeFix.Testing.XUnit", - "Microsoft.CodeAnalysis.CSharp.CodeRefactoring.Testing", - "Microsoft.CodeAnalysis.CSharp.CodeRefactoring.Testing.MSTest", - "Microsoft.CodeAnalysis.CSharp.CodeRefactoring.Testing.NUnit", - "Microsoft.CodeAnalysis.CSharp.CodeRefactoring.Testing.XUnit", - "Microsoft.CodeAnalysis.CSharp.SourceGenerators.Testing", - "Microsoft.CodeAnalysis.CSharp.SourceGenerators.Testing.MSTest", - "Microsoft.CodeAnalysis.CSharp.SourceGenerators.Testing.NUnit", - "Microsoft.CodeAnalysis.CSharp.SourceGenerators.Testing.XUnit", - "Microsoft.CodeAnalysis.SourceGenerators.Testing", - "Microsoft.CodeAnalysis.Testing.Verifiers.MSTest", - "Microsoft.CodeAnalysis.Testing.Verifiers.NUnit", - "Microsoft.CodeAnalysis.Testing.Verifiers.XUnit", - "Microsoft.CodeAnalysis.VisualBasic.Analyzer.Testing", - "Microsoft.CodeAnalysis.VisualBasic.Analyzer.Testing.MSTest", - "Microsoft.CodeAnalysis.VisualBasic.Analyzer.Testing.NUnit", - "Microsoft.CodeAnalysis.VisualBasic.Analyzer.Testing.XUnit", - "Microsoft.CodeAnalysis.VisualBasic.CodeFix.Testing", - "Microsoft.CodeAnalysis.VisualBasic.CodeFix.Testing.MSTest", - "Microsoft.CodeAnalysis.VisualBasic.CodeFix.Testing.NUnit", - "Microsoft.CodeAnalysis.VisualBasic.CodeFix.Testing.XUnit", - "Microsoft.CodeAnalysis.VisualBasic.CodeRefactoring.Testing", - "Microsoft.CodeAnalysis.VisualBasic.CodeRefactoring.Testing.MSTest", - "Microsoft.CodeAnalysis.VisualBasic.CodeRefactoring.Testing.NUnit", - "Microsoft.CodeAnalysis.VisualBasic.CodeRefactoring.Testing.XUnit", - "Microsoft.CodeAnalysis.VisualBasic.SourceGenerators.Testing", - "Microsoft.CodeAnalysis.VisualBasic.SourceGenerators.Testing.MSTest", - "Microsoft.CodeAnalysis.VisualBasic.SourceGenerators.Testing.NUnit", - "Microsoft.CodeAnalysis.VisualBasic.SourceGenerators.Testing.XUnit" - }; - - internal static async Task PublishAsync(string repoName, string source, string apiKey, bool unlisted, ILogger logger) + internal static async Task PublishAsync(string repoName, string source, string apiKey, bool unlisted, bool ignoreMissingPackage, ILogger logger) { try { @@ -96,12 +31,37 @@ internal static async Task PublishAsync(string repoName, string source, str return 1; } + Contract.Assert(version is not null); + var packageIds = repoName == RoslynRepo ? RoslynPackageIds : RoslynSdkPackageIds; logger.LogInformation($"Publishing {version} packages..."); + var hasMissingPackageError = false; + foreach (var packageId in packageIds) + { + if (!File.Exists(GetPackageFileName(packageId, version))) + { + if (ignoreMissingPackage) + { + logger.LogInformation($"Missing package '{packageId}' is ignored."); + } + else + { + logger.LogError($"Required package '{packageId}' is missing"); + hasMissingPackageError = true; + } + } + } + + if (hasMissingPackageError) + { + logger.LogInformation("Publication failed."); + return 1; + } + foreach (var packageId in packageIds) { var result = await PublishPackageAsync(packageId, version); @@ -132,38 +92,12 @@ internal static async Task PublishAsync(string repoName, string source, str return 0; - static bool TryDetermineRoslynPackageVersion([NotNullWhen(returnValue: true)] out string? version) - { - var packageFileName = Directory.GetFiles(Environment.CurrentDirectory, "Microsoft.Net.Compilers.Toolset.*.nupkg").FirstOrDefault(); - if (packageFileName is null) - { - version = null; - return false; - } - - version = Path.GetFileNameWithoutExtension(packageFileName).Substring(32); - return true; - } - - static bool TryDetermineRoslynSdkPackageVersion([NotNullWhen(returnValue: true)] out string? version) - { - var packageFileName = Directory.GetFiles(Environment.CurrentDirectory, "Microsoft.CodeAnalysis.Analyzer.Testing.*.nupkg").FirstOrDefault(); - if (packageFileName is null) - { - version = null; - return false; - } - - version = Path.GetFileNameWithoutExtension(packageFileName).Substring(40); - return true; - } - - Task PublishPackageAsync(string packageId, string? version) + Task PublishPackageAsync(string packageId, string version) { - return ProcessRunner.RunProcessAsync("dotnet", $"nuget push --source \"{source}\" --api-key \"{apiKey}\" \"{packageId}.{version}.nupkg\""); + return ProcessRunner.RunProcessAsync("dotnet", $"nuget push --source \"{source}\" --api-key \"{apiKey}\" \"{GetPackageFileName(packageId, version)}\""); } - Task UnlistPackageAsync(string packageId, string? version) + Task UnlistPackageAsync(string packageId, string version) { return ProcessRunner.RunProcessAsync("dotnet", $"nuget delete {packageId} {version} --source \"{source}\" --api-key \"{apiKey}\" --non-interactive"); } From 4953d2b575003a63c7ecad3bb1c7066178a90673 Mon Sep 17 00:00:00 2001 From: "gel@microsoft.com" Date: Wed, 1 Mar 2023 14:41:18 -0800 Subject: [PATCH 2/2] Display all dependents of dependencies need attention --- .../NuGet/NuGetDependencyFinder.cs | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/dotnet-roslyn-tools/NuGet/NuGetDependencyFinder.cs b/src/dotnet-roslyn-tools/NuGet/NuGetDependencyFinder.cs index cba6ad03ed..afcc56be7f 100644 --- a/src/dotnet-roslyn-tools/NuGet/NuGetDependencyFinder.cs +++ b/src/dotnet-roslyn-tools/NuGet/NuGetDependencyFinder.cs @@ -44,7 +44,8 @@ where regex.Success var dependencies = new Dictionary>(); - await foreach (var dependency in GetAllDependenciesAsync()) + var (foundDependencies, idToDependentsMap) = await GetAllDependenciesAsync().ConfigureAwait(false); + foreach (var dependency in foundDependencies) { DependencyResult result; NuGetVersion? desiredVersion = null; @@ -125,6 +126,10 @@ where v.Version > dependency.VersionRange.MinVersion.Version foreach (var (dependency, _) in releaseUnavailablePackages.OrderBy(x => x.Dependency.Id)) { logger.LogWarning("{DependencyId}, {DependencyMinVersion}", dependency.Id, dependency.VersionRange.MinVersion); + + logger.LogWarning("Dependents:"); + foreach (var dependent in idToDependentsMap[dependency.Id]) + logger.LogWarning($" {dependent}"); } } @@ -135,6 +140,10 @@ where v.Version > dependency.VersionRange.MinVersion.Version foreach (var (dependency, _) in missingPackages.OrderBy(x => x.Dependency.Id)) { logger.LogError("{DependencyId}, {DependencyMinVersion}", dependency.Id, dependency.VersionRange.MinVersion); + + logger.LogError("Dependents:"); + foreach (var dependent in idToDependentsMap[dependency.Id]) + logger.LogError($" {dependent}"); } } @@ -146,11 +155,12 @@ where v.Version > dependency.VersionRange.MinVersion.Version return 1; } - async IAsyncEnumerable GetAllDependenciesAsync() + async Task<(List foundDependencies, Dictionary> idToDependentsMap)> GetAllDependenciesAsync() { var local = Repository.Factory.GetCoreV3(packageFolder); var localFinder = await local.GetResourceAsync().ConfigureAwait(false); - var foundPackages = new HashSet(); + var idToDependentsMap = new Dictionary>(); + var foundDependencies = new List(); foreach (var package in packages) { @@ -161,13 +171,19 @@ async IAsyncEnumerable GetAllDependenciesAsync() { foreach (var dependency in group.Packages) { - if (foundPackages.Add(dependency.Id)) + if (!idToDependentsMap.TryGetValue(dependency.Id, out var dependentsList)) { - yield return dependency; + foundDependencies.Add(dependency); + dependentsList = new List(); + idToDependentsMap[dependency.Id] = dependentsList; } + + dependentsList.Add(package); } } } + + return (foundDependencies, idToDependentsMap); } } }