Rust logs collection

All.sln

@@ -132,6 +132,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "misc", "misc", "{2918C7E3-3
 		Ficus\Readme.md = Ficus\Readme.md
 	EndProjectSection
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Salve", "Salve", "{BA9BD6EC-204D-40AA-86A7-E70087EB9A1D}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Salve", "Salve\Salve.csproj", "{11FB9788-D067-4AD5-A17D-4685DE9DD366}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -238,6 +242,10 @@ Global
 		{1BBDEB95-1084-4981-AE5A-EFFC74E095E4}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{1BBDEB95-1084-4981-AE5A-EFFC74E095E4}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{1BBDEB95-1084-4981-AE5A-EFFC74E095E4}.Release|Any CPU.Build.0 = Release|Any CPU
+		{11FB9788-D067-4AD5-A17D-4685DE9DD366}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{11FB9788-D067-4AD5-A17D-4685DE9DD366}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{11FB9788-D067-4AD5-A17D-4685DE9DD366}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{11FB9788-D067-4AD5-A17D-4685DE9DD366}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{FEF48EF9-EDFC-4464-8942-64FC9FF000CA} = {C3E829BA-4C31-4F55-BF4D-C0D11B0B70B4}
@@ -277,5 +285,6 @@ Global
 		{B1223284-182E-4C3A-92CD-F59FBD42FFD6} = {0EC07BD3-AB92-48A0-B68B-05B3F2A767A3}
 		{1BBDEB95-1084-4981-AE5A-EFFC74E095E4} = {D50BD31E-296B-468E-817A-60AF1CE7A759}
 		{2918C7E3-3FC8-48A5-ADDF-8358E888B40C} = {5F2E0AEA-6AAF-4130-B486-A5F4F5A8A4BD}
+		{11FB9788-D067-4AD5-A17D-4685DE9DD366} = {BA9BD6EC-204D-40AA-86A7-E70087EB9A1D}
 	EndGlobalSection
 EndGlobal

Directory.Packages.props

@@ -4,7 +4,9 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageVersion Include="Aspire.Hosting" Version="9.3.0.0" />
+    <PackageVersion Include="Dbscan" Version="3.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Logging.Configuration" Version="10.0.0" />
+    <PackageVersion Include="Spectre.Console.Cli" Version="1.0.0-alpha.0.12" />
     <PackageVersion Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
     <PackageVersion Include="Confluent.Kafka" Version="2.5.2" />
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.6.0" />

Salve/ClusteringUtils.cs

@@ -0,0 +1,102 @@
+using System.Numerics;
+
+namespace Salve;
+
+internal static class ClusteringUtils
+{
+  public static int CalculateEditDistance<T>(ReadOnlySpan<T> first, ReadOnlySpan<T> second) where T : IEqualityOperators<T, T, bool>
+  {
+    if (first.Length == 0) return second.Length;
+    if (second.Length == 0) return first.Length;
+
+    var current = 1;
+    var previous = 0;
+
+    var r = new int[2, second.Length + 1];
+    for (var i = 0; i <= second.Length; i++)
+    {
+      r[previous, i] = i;
+    }
+
+    for (var i = 0; i < first.Length; i++)
+    {
+      r[current, 0] = i + 1;
+      for (var j = 1; j <= second.Length; j++)
+      {
+        var cost = (second[j - 1] == first[i]) ? 0 : 1;
+        r[current, j] = Min(r[previous, j] + 1, r[current, j - 1] + 1, r[previous, j - 1] + cost);
+      }
+
+      previous = (previous + 1) % 2;
+      current = (current + 1) % 2;
+    }
+
+    return r[previous, second.Length];
+  }
+
+  private static int Min(int e1, int e2, int e3) => Math.Min(Math.Min(e1, e2), e3);
+
+  public record struct LcsInfo<T>(T[] Lcs, List<int> FirstIndices, List<int> SecondIndices);
+
+  public static LcsInfo<T> FindLcs<T>(ReadOnlySpan<T> first, ReadOnlySpan<T> second)
+    where T : IEqualityOperators<T, T, bool>
+  {
+    var n = first.Length;
+    var m = second.Length;
+    var dp = new int[n + 1, m + 1];
+
+    for (var i = 1; i <= n; i++)
+    {
+      for (var j = 1; j <= m; j++)
+      {
+        if (first[i - 1] == second[j - 1])
+        {
+          dp[i, j] = dp[i - 1, j - 1] + 1;
+        }
+        else
+        {
+          dp[i, j] = Math.Max(dp[i - 1, j], dp[i, j - 1]);
+        }
+      }
+    }
+
+    return RestoreLcs(first, second, dp, n, m);
+  }
+
+  private static LcsInfo<T> RestoreLcs<T>(ReadOnlySpan<T> first, ReadOnlySpan<T> second, int[,] dp, int n, int m)
+    where T : IEqualityOperators<T, T, bool>
+  {
+    int i = n, j = m;
+    List<T> lcs = [];
+    List<int> firstIndices = [];
+    List<int> secondIndices = [];
+
+    while (i > 0 && j > 0)
+    {
+      if (first[i - 1] == second[j - 1])
+      {
+        firstIndices.Add(i - 1);
+        secondIndices.Add(j - 1);
+
+        lcs.Add(first[i - 1]);
+
+        i--;
+        j--;
+      }
+      else if (dp[i - 1, j] > dp[i, j - 1])
+      {
+        i--;
+      }
+      else
+      {
+        j--;
+      }
+    }
+
+    firstIndices.Reverse();
+    secondIndices.Reverse();
+    lcs.Reverse();
+
+    return new LcsInfo<T>(lcs.ToArray(), firstIndices, secondIndices);
+  }
+}

Salve/Program.cs

@@ -0,0 +1,114 @@
+using System.ComponentModel;
+using System.Diagnostics;
+using Bxes.Utils;
+using JetBrains.Annotations;
+using Salve.Rust;
+using Spectre.Console;
+using Spectre.Console.Cli;
+
+var app = new CommandApp();
+app.Configure(cfg => { cfg.AddCommand<RustcLogsToBxes>("rustc-logs-to-bxes"); });
+
+app.Run(args);
+
+
+[UsedImplicitly]
+internal class RustcLogsToBxes : Command<RustcLogsToBxes.Settings>
+{
+  [UsedImplicitly]
+  public class Settings : CommandSettings
+  {
+    [CommandArgument(1, "<o>")]
+    [Description("The output path of a bXES file")]
+    public required string OutputFilePath { get; init; }
+
+    [CommandOption("--args")]
+    [Description("Command arguments")]
+    // ReSharper disable once UnassignedGetOnlyAutoProperty
+    public string? Arguments { get; init; }
+
+    [CommandOption("--workdir")]
+    [Description("Working directory")]
+    // ReSharper disable once UnassignedGetOnlyAutoProperty
+    public string? WorkingDirectory { get; init; }
+
+    [CommandOption("--group-names-as-event-names")]
+    [Description("Use groups names (FQNs) as event names")]
+    // ReSharper disable once UnassignedGetOnlyAutoProperty
+    public bool UseGroupsAsEventNames { get; init; }
+
+    [CommandOption("--max-tokens-in-event")]
+    [Description("Maximum tokens in events")]
+    public int MaxTokensInEvent { get; init; } = 10;
+
+    [CommandOption("--leave-only-method-events")]
+    [Description("Leave only methods tracing events")]
+    public bool LeaveOnlyMethodEvents { get; init; }
+
+
+    public RustcLogsParser CreateProcessor() =>
+      new(OutputFilePath, UseGroupsAsEventNames, MaxTokensInEvent, LeaveOnlyMethodEvents);
+  }
+
+
+  protected override int Execute(CommandContext context, Settings settings, CancellationToken cancellationToken)
+  {
+    try
+    {
+      var directory = Path.GetDirectoryName(settings.OutputFilePath);
+      if (!Directory.Exists(directory))
+      {
+        throw new Exception($"Directory {directory} does not exist");
+      }
+
+      PathUtil.EnsureDeleted(settings.OutputFilePath);
+
+      var info = new ProcessStartInfo
+      {
+        FileName = "rustc",
+        RedirectStandardOutput = true,
+        RedirectStandardError = true,
+        WorkingDirectory = settings.WorkingDirectory,
+        Arguments = settings.Arguments,
+        CreateNoWindow = true
+      };
+
+      var process = new Process
+      {
+        StartInfo = info
+      };
+
+      var processor = settings.CreateProcessor();
+      processor.Initialize();
+
+      try
+      {
+        // ReSharper disable once AccessToDisposedClosure
+        process.OutputDataReceived += (_, args) => processor.Process(args.Data);
+        // ReSharper disable once AccessToDisposedClosure
+        process.ErrorDataReceived += (_, args) => processor.Process(args.Data);
+
+        if (!process.Start())
+        {
+          throw new Exception("Failed to start process");
+        }
+
+        process.BeginOutputReadLine();
+        process.BeginErrorReadLine();
+
+        process.WaitForExit();
+      }
+      finally
+      {
+        processor.Dispose();
+      }
+
+      return 0;
+    }
+    catch (Exception ex)
+    {
+      AnsiConsole.WriteException(ex);
+      return 1;
+    }
+  }
+}

Salve/Rust/RustcLogsParser.EventIndex.cs

@@ -0,0 +1,69 @@
+using Dbscan;
+
+namespace Salve.Rust;
+
+internal partial class RustcLogsParser
+{
+  private enum EventKind
+  {
+    Message,
+    Method
+  }
+
+  private class Event(EventKind kind, string message, string group) : IPointData
+  {
+    public EventKind Kind => kind;
+    public string Name { get; set; } = message;
+    public string Group => group;
+    public Point Point => default;
+  }
+
+  private record EventWithTokens(Event Event, int[] Tokens) : IPointData
+  {
+    public Point Point => Event.Point;
+  }
+
+  private class EventsIndex(List<EventWithTokens> events) : ISpatialIndex<PointInfo<EventWithTokens>>
+  {
+    private readonly Dictionary<string, List<PointInfo<EventWithTokens>>> myEventsByGroups =
+      events
+        .GroupBy(e => e.Event.Group)
+        .ToDictionary(
+          e => e.Key,
+          e => e
+            .Select(evt => new PointInfo<EventWithTokens>(evt))
+            .ToList()
+        );
+
+
+    public IReadOnlyList<PointInfo<EventWithTokens>> Search() =>
+      myEventsByGroups.Values.SelectMany(v => v).Where(e => ShouldCluster(e.Item.Event)).ToList();
+
+    private static bool ShouldCluster(Event e) => e.Kind is EventKind.Message;
+
+    public IReadOnlyList<PointInfo<EventWithTokens>> Search(in IPointData p, double epsilon)
+    {
+      var point = (PointInfo<EventWithTokens>)p;
+      var result = new List<PointInfo<EventWithTokens>>();
+
+      foreach (var evt in myEventsByGroups[point.Item.Event.Group])
+      {
+        if (!ShouldCluster(evt.Item.Event) || ReferenceEquals(evt, point))
+        {
+          continue;
+        }
+
+        if (point.Item.Tokens.Length != evt.Item.Tokens.Length) continue;
+
+        var distance = ClusteringUtils.CalculateEditDistance(point.Item.Tokens, evt.Item.Tokens);
+
+        if (distance <= epsilon)
+        {
+          result.Add(evt);
+        }
+      }
+
+      return result;
+    }
+  }
+}