Merge pull request #418 from WildernessLabs/linker

Linker refactor and v2 refresh

Source/TestApps/LinkerTest/LinkerTest.sln

@@ -0,0 +1,25 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.8.34309.116
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LinkerTest", "LinkerTest\LinkerTest.csproj", "{F18B502F-1D67-4F9A-8F1F-6A3C91C942E9}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{F18B502F-1D67-4F9A-8F1F-6A3C91C942E9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F18B502F-1D67-4F9A-8F1F-6A3C91C942E9}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F18B502F-1D67-4F9A-8F1F-6A3C91C942E9}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F18B502F-1D67-4F9A-8F1F-6A3C91C942E9}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {8AF82077-F626-42F0-81B2-B92439C770DA}
+	EndGlobalSection
+EndGlobal

Source/TestApps/LinkerTest/LinkerTest/ILLinker.cs

@@ -0,0 +1,86 @@
+using Microsoft.Extensions.Logging;
+using System.Diagnostics;
+
+namespace LinkerTest
+{
+    internal class ILLinker
+    {
+        readonly ILogger? _logger;
+
+        public ILLinker(ILogger? logger = null)
+        {
+            _logger = logger;
+        }
+
+        public async Task RunILLink(
+            string illinkerDllPath,
+            string descriptorXmlPath,
+            string noLinkArgs,
+            string prelinkAppPath,
+            string prelinkDir,
+            string postlinkDir)
+        {
+            if (!File.Exists(illinkerDllPath))
+            {
+                throw new FileNotFoundException("Cannot run trimming operation, illink.dll not found");
+            }
+
+            //original
+            //var monolinker_args = $"\"{illinkerDllPath}\" -x \"{descriptorXmlPath}\" {noLinkArgs} --skip-unresolved --deterministic --keep-facades true --ignore-descriptors true -b true -c link -o \"{postlinkDir}\" -r \"{prelinkAppPath}\" -a \"{prelink_os}\" -d \"{prelinkDir}\"";
+
+            var monolinker_args = $"\"{illinkerDllPath}\"" +
+                $" -x \"{descriptorXmlPath}\" " + //link files in the descriptor file (needed)
+                $"{noLinkArgs} " + //arguments to skip linking - will be blank if we are linking
+                $"-r \"{prelinkAppPath}\" " + //link the app in the prelink folder (needed)
+                $"--skip-unresolved true " + //skip unresolved references (needed -hangs without)
+                $"--deterministic true " + //make deterministic (to avoid pushing unchanged files to the device)
+                $"--keep-facades true " + //keep facades (needed - will skip key libs without)
+                $"-b true " + //Update debug symbols for each linked module (needed - will skip key libs without)
+                $"-o \"{postlinkDir}\" " + //output directory
+
+
+                //old
+                //$"--ignore-descriptors false " + //ignore descriptors (doesn't appear to impact behavior)
+                //$"-c link " + //link framework assemblies
+                //$"-d \"{prelinkDir}\"" //additional folder to link (not needed)
+
+                //experimental
+                //$"--explicit-reflection true " + //enable explicit reflection (throws an exception with it)
+                //$"--keep-dep-attributes true " + //keep dependency attributes (files are slightly larger with, doesn't fix dependency issue)
+                "";
+
+            _logger?.Log(LogLevel.Information, "Trimming assemblies");
+
+            using (var process = new Process())
+            {
+                process.StartInfo.WorkingDirectory = Directory.GetDirectoryRoot(illinkerDllPath);
+                process.StartInfo.FileName = "dotnet";
+                process.StartInfo.Arguments = monolinker_args;
+                process.StartInfo.UseShellExecute = false;
+                process.StartInfo.CreateNoWindow = true;
+                process.StartInfo.RedirectStandardError = true;
+                process.StartInfo.RedirectStandardOutput = true;
+                process.Start();
+
+                // To avoid deadlocks, read the output stream first and then wait
+                string stdOutReaderResult;
+                using (StreamReader stdOutReader = process.StandardOutput)
+                {
+                    stdOutReaderResult = await stdOutReader.ReadToEndAsync();
+
+                    Console.WriteLine("StandardOutput Contains: " + stdOutReaderResult);
+
+                    _logger?.Log(LogLevel.Debug, "StandardOutput Contains: " + stdOutReaderResult);
+                }
+
+                await process.WaitForExitAsync();
+
+                if (process.ExitCode != 0)
+                {
+                    _logger?.Log(LogLevel.Debug, $"Trimming failed - ILLinker execution error!\nProcess Info: {process.StartInfo.FileName} {process.StartInfo.Arguments} \nExit Code: {process.ExitCode}");
+                    throw new Exception("Trimming failed");
+                }
+            }
+        }
+    }
+}

Source/TestApps/LinkerTest/LinkerTest/LinkerTest.csproj

@@ -0,0 +1,47 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+	<LangVersion>preview</LangVersion>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
+    <PackageReference Include="Mono.Cecil" Version="0.11.2" />
+  </ItemGroup>
+
+	<ItemGroup>
+		<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+		<PackageReference Include="CliFx" Version="2.3.4" />
+	</ItemGroup>
+
+
+	<ItemGroup>
+		<None Update="appsettings.json">
+			<CopyToOutputDirectory>Always</CopyToOutputDirectory>
+		</None>
+		<None Update="lib\illink.deps.json">
+			<CopyToOutputDirectory>Always</CopyToOutputDirectory>
+		</None>
+		<None Update="lib\illink.dll">
+			<CopyToOutputDirectory>Always</CopyToOutputDirectory>
+		</None>
+		<None Update="lib\illink.runtimeconfig.json">
+			<CopyToOutputDirectory>Always</CopyToOutputDirectory>
+		</None>
+		<None Update="lib\meadow_link.xml">
+			<CopyToOutputDirectory>Always</CopyToOutputDirectory>
+		</None>
+		<None Update="lib\Mono.Cecil.dll">
+		  <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+		</None>
+		<None Update="lib\Mono.Cecil.Pdb.dll">
+		  <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+		</None>
+	</ItemGroup>
+
+</Project>

Source/TestApps/LinkerTest/LinkerTest/MeadowLinker.cs

@@ -0,0 +1,175 @@
+using Microsoft.Extensions.Logging;
+using Mono.Cecil;
+using Mono.Collections.Generic;
+using System.Reflection;
+
+namespace LinkerTest;
+
+public class MeadowLinker(string meadowAssembliesPath, ILogger? logger = null)
+{
+    private const string IL_LINKER_DIR = "lib";
+    private const string IL_LINKER_DLL = "illink.dll";
+    private const string MEADOW_LINK_XML = "meadow_link.xml";
+
+    public const string PostLinkDirectoryName = "postlink_bin";
+    public const string PreLinkDirectoryName = "prelink_bin";
+
+    readonly ILLinker _linker = new ILLinker(logger);
+    readonly ILogger? _logger = logger;
+
+    private readonly string _meadowAssembliesPath = meadowAssembliesPath;
+
+    public async Task Trim(
+        FileInfo meadowAppFile,
+        bool includePdbs = false,
+        IList<string>? noLink = null)
+    {
+        var dependencies = MapDependencies(meadowAppFile);
+
+        CopyDependenciesToPreLinkFolder(meadowAppFile, dependencies, includePdbs);
+
+        //run the _linker against the dependencies
+        await TrimMeadowApp(meadowAppFile, noLink);
+    }
+
+    List<string> MapDependencies(FileInfo meadowAppFile)
+    {
+        //get all dependencies in meadowAppFile and exclude the Meadow App 
+        var dependencyMap = new List<string>();
+
+        var appRefs = GetAssemblyReferences(meadowAppFile.FullName);
+        return GetDependencies(meadowAppFile.FullName, appRefs, dependencyMap, meadowAppFile.DirectoryName);
+    }
+
+    public void CopyDependenciesToPreLinkFolder(
+        FileInfo meadowApp,
+        List<string> dependencies,
+        bool includePdbs)
+    {
+        //set up the paths
+        var prelinkDir = Path.Combine(meadowApp.DirectoryName!, PreLinkDirectoryName);
+        var postlinkDir = Path.Combine(meadowApp.DirectoryName!, PostLinkDirectoryName);
+
+        //create output directories
+        CreateEmptyDirectory(prelinkDir);
+        CreateEmptyDirectory(postlinkDir);
+
+        //copy meadow app
+        File.Copy(meadowApp.FullName, Path.Combine(prelinkDir, meadowApp.Name), overwrite: true);
+
+        //copy dependencies and optional pdbs from the local folder and the meadow assemblies folder
+        foreach (var dependency in dependencies)
+        {
+            var destination = Path.Combine(prelinkDir, Path.GetFileName(dependency));
+            File.Copy(dependency, destination, overwrite: true);
+
+            if (includePdbs)
+            {
+                var pdbFile = Path.ChangeExtension(dependency, "pdb");
+                if (File.Exists(pdbFile))
+                {
+                    destination = Path.ChangeExtension(destination, "pdb");
+                    File.Copy(pdbFile, destination, overwrite: true);
+                }
+            }
+        }
+    }
+
+    public async Task<IEnumerable<string>?> TrimMeadowApp(
+        FileInfo file,
+        IList<string>? noLink)
+    {
+        //set up the paths
+        var prelink_dir = Path.Combine(file.DirectoryName!, PreLinkDirectoryName);
+        var postlink_dir = Path.Combine(file.DirectoryName!, PostLinkDirectoryName);
+        var prelink_app = Path.Combine(prelink_dir, file.Name);
+        var base_path = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
+        var illinker_path = Path.Combine(base_path!, IL_LINKER_DIR, IL_LINKER_DLL);
+        var descriptor_path = Path.Combine(base_path!, IL_LINKER_DIR, MEADOW_LINK_XML);
+
+        //prepare _linker arguments
+        var no_link_args = noLink != null ? string.Join(" ", noLink.Select(o => $"-p copy \"{o}\"")) : string.Empty;
+
+        try
+        {
+            //link the apps
+            await _linker.RunILLink(illinker_path, descriptor_path, no_link_args, prelink_app, prelink_dir, postlink_dir);
+        }
+        catch (Exception ex)
+        {
+            _logger?.LogError(ex, "Error trimming Meadow app");
+        }
+
+        return Directory.EnumerateFiles(postlink_dir);
+    }
+
+    /// <summary>
+    /// This method recursively gets all dependencies for the given assembly
+    /// </summary>
+    private List<string> GetDependencies(string assemblyPath, Collection<AssemblyNameReference> assemblyReferences, List<string> dependencyMap, string appDir)
+    {
+        if (dependencyMap.Contains(assemblyPath))
+        {   //already have this assembly mapped
+            return dependencyMap;
+        }
+
+        dependencyMap.Add(assemblyPath);
+
+        foreach (var reference in assemblyReferences)
+        {
+            var fullPath = FindAssemblyFullPath(reference.Name, appDir, _meadowAssembliesPath);
+
+            Collection<AssemblyNameReference> namedRefs = default!;
+
+            if (fullPath == null)
+            {
+                continue;
+            }
+            namedRefs = GetAssemblyReferences(fullPath);
+
+            //recursive!
+            dependencyMap = GetDependencies(fullPath!, namedRefs!, dependencyMap, appDir);
+        }
+
+        return dependencyMap.Where(x => x.Contains("App.") == false).ToList();
+    }
+
+    static string? FindAssemblyFullPath(string fileName, string localPath, string meadowAssembliesPath)
+    {
+        //Assembly may not have a file extension, add .dll if it doesn't
+        if (Path.GetExtension(fileName) != ".exe" &&
+            Path.GetExtension(fileName) != ".dll")
+        {
+            fileName += ".dll";
+        }
+
+        //meadow assemblies path
+        if (File.Exists(Path.Combine(meadowAssembliesPath, fileName)))
+        {
+            return Path.Combine(meadowAssembliesPath, fileName);
+        }
+
+        //localPath
+        if (File.Exists(Path.Combine(localPath, fileName)))
+        {
+            return Path.Combine(localPath, fileName);
+        }
+
+        return null;
+    }
+
+    private Collection<AssemblyNameReference> GetAssemblyReferences(string assemblyPath)
+    {
+        using var definition = AssemblyDefinition.ReadAssembly(assemblyPath);
+        return definition.MainModule.AssemblyReferences;
+    }
+
+    private void CreateEmptyDirectory(string directoryPath)
+    {
+        if (Directory.Exists(directoryPath))
+        {
+            Directory.Delete(directoryPath, recursive: true);
+        }
+        Directory.CreateDirectory(directoryPath);
+    }
+}

Source/TestApps/LinkerTest/LinkerTest/Program.cs

@@ -0,0 +1,50 @@
+using System.Diagnostics;
+
+namespace LinkerTest
+{
+    internal class Program
+    {
+        private static readonly string _meadowAssembliesPath = @"C:\Users\adria\AppData\Local\WildernessLabs\Firmware\1.6.0.1\meadow_assemblies\";
+
+        static async Task Main(string[] args)
+        {
+            Console.WriteLine("Hello, World!");
+
+            //  await OtherLink();
+
+            //  return;
+
+            var linker = new MeadowLinker(_meadowAssembliesPath);
+
+            string fileToLink = @"H:\WL\Meadow.ProjectLab\Source\ProjectLab_Demo\bin\Debug\netstandard2.1\App.dll";
+
+            await linker.Trim(new FileInfo(fileToLink), true);
+        }
+
+        static async Task OtherLink()
+        {
+            var monolinker_args = @"""H:\WL\Meadow.CLI\Meadow.CLI.Classic\bin\Debug\lib\illink.dll"" -x ""H:\WL\Meadow.CLI\Meadow.CLI.Classic\bin\Debug\lib\meadow_link.xml""   --skip-unresolved --deterministic --keep-facades true --ignore-descriptors true -b true -c link -o ""H:\WL\Meadow.ProjectLab\Source\ProjectLab_Demo\bin\Debug\netstandard2.1\postlink_bin"" -r ""H:\WL\Meadow.ProjectLab\Source\ProjectLab_Demo\bin\Debug\netstandard2.1\prelink_bin\App.dll"" -a ""H:\WL\Meadow.ProjectLab\Source\ProjectLab_Demo\bin\Debug\netstandard2.1\prelink_bin\Meadow.dll"" -d ""H:\WL\Meadow.ProjectLab\Source\ProjectLab_Demo\bin\Debug\netstandard2.1\prelink_bin""";
+
+            Console.WriteLine("Trimming assemblies to reduce size (may take several seconds)...");
+
+            using (var process = new Process())
+            {
+                process.StartInfo.FileName = "dotnet";
+                process.StartInfo.Arguments = monolinker_args;
+                process.StartInfo.UseShellExecute = false;
+                process.StartInfo.CreateNoWindow = true;
+                process.StartInfo.RedirectStandardError = true;
+                process.StartInfo.RedirectStandardOutput = true;
+                process.Start();
+
+                // To avoid deadlocks, read the output stream first and then wait
+                string stdOutReaderResult;
+                using (StreamReader stdOutReader = process.StandardOutput)
+                {
+                    stdOutReaderResult = await stdOutReader.ReadToEndAsync();
+                    Console.WriteLine("StandardOutput Contains: " + stdOutReaderResult);
+                }
+            }
+        }
+    }
+}