Merge pull request #43 from Numpsy/rw/stj1

Parse grammar definitions with System.Text.Json

build/Directory.Build.props

@@ -2,6 +2,6 @@
   <PropertyGroup>
     <LangVersion>latest</LangVersion>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
-    <NewtonsoftJsonVersion>13.0.2</NewtonsoftJsonVersion>
+    <SystemTextJsonVersion>7.0.2</SystemTextJsonVersion>
   </PropertyGroup>
 </Project>

src/TextMateSharp.Grammars/GrammarDefinition.cs

@@ -1,30 +1,29 @@
 using System.Collections.Generic;
-
-using Newtonsoft.Json;
+using System.Text.Json.Serialization;
 
 namespace TextMateSharp.Grammars
 {
     public class Engines
     {
-        [JsonProperty("engines")]
+        [JsonPropertyName("engines")]
         public string VsCode { get; set; }
     }
 
     public class Scripts
     {
-        [JsonProperty("update-grammar")]
+        [JsonPropertyName("update-grammar")]
         public string UpdateGrammar { get; set; }
     }
 
     public class Language
     {
-        [JsonProperty("id")]
+        [JsonPropertyName("id")]
         public string Id { get; set; }
-        [JsonProperty("extensions")]
+        [JsonPropertyName("extensions")]
         public List<string> Extensions { get; set; }
-        [JsonProperty("aliases")]
+        [JsonPropertyName("aliases")]
         public List<string> Aliases { get; set; }
-        [JsonProperty("configuration")]
+        [JsonPropertyName("configuration")]
         public string Configuration { get; set; }
 
         public override string ToString()
@@ -38,61 +37,61 @@ public override string ToString()
 
     public class Grammar
     {
-        [JsonProperty("language")]
+        [JsonPropertyName("language")]
         public string Language { get; set; }
-        [JsonProperty("scopeName")]
+        [JsonPropertyName("scopeName")]
         public string ScopeName { get; set; }
-        [JsonProperty("path")]
+        [JsonPropertyName("path")]
         public string Path { get; set; }
     }
 
     public class Snippet
     {
-        [JsonProperty("language")]
+        [JsonPropertyName("language")]
         public string Language { get; set; }
-        [JsonProperty("path")]
+        [JsonPropertyName("path")]
         public string Path { get; set; }
     }
 
     public class Contributes
     {
-        [JsonProperty("languages")]
+        [JsonPropertyName("languages")]
         public List<Language> Languages { get; set; }
-        [JsonProperty("grammars")]
+        [JsonPropertyName("grammars")]
         public List<Grammar> Grammars { get; set; }
-        [JsonProperty("snippets")]
+        [JsonPropertyName("snippets")]
         public List<Snippet> Snippets { get; set; }
     }
 
     public class Repository
     {
-        [JsonProperty("type")]
+        [JsonPropertyName("type")]
         public string Type { get; set; }
-        [JsonProperty("url")]
+        [JsonPropertyName("url")]
         public string Url { get; set; }
     }
 
     public class GrammarDefinition
     {
-        [JsonProperty("name")]
+        [JsonPropertyName("name")]
         public string Name { get; set; }
-        [JsonProperty("displayName")]
+        [JsonPropertyName("displayName")]
         public string DisplayName { get; set; }
-        [JsonProperty("description")]
+        [JsonPropertyName("description")]
         public string Description { get; set; }
-        [JsonProperty("version")]
+        [JsonPropertyName("version")]
         public string Version { get; set; }
-        [JsonProperty("publisher")]
+        [JsonPropertyName("publisher")]
         public string Publisher { get; set; }
-        [JsonProperty("license")]
+        [JsonPropertyName("license")]
         public string License { get; set; }
-        [JsonProperty("engines")]
+        [JsonPropertyName("engines")]
         public Engines Engines { get; set; }
-        [JsonProperty("scripts")]
+        [JsonPropertyName("scripts")]
         public Scripts Scripts { get; set; }
-        [JsonProperty("contributes")]
+        [JsonPropertyName("contributes")]
         public Contributes Contributes { get; set; }
-        [JsonProperty("repository")]
+        [JsonPropertyName("repository")]
         public Repository Repository { get; set; }
     }
 }

src/TextMateSharp.Grammars/RegistryOptions.cs

@@ -1,11 +1,10 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
-
+using System.Text.Json;
+using System.Text.Json.Serialization;
 using TextMateSharp.Grammars.Resources;
 
-using Newtonsoft.Json;
-
 using TextMateSharp.Internal.Grammars.Reader;
 using TextMateSharp.Internal.Themes.Reader;
 using TextMateSharp.Internal.Types;
@@ -159,37 +158,16 @@ public IRawTheme GetDefaultTheme()
 
         void InitializeAvailableGrammars()
         {
-            KeepType<List<object>>();
-            KeepType<GrammarDefinition>();
-            KeepType<Repository>();
-            KeepType<Contributes>();
-            KeepType<Snippet>();
-            KeepType<Grammar>();
-            KeepType<Language>();
-            KeepType<Scripts>();
-            KeepType<Engines>();
-
-            var serializer = new JsonSerializer();
-
             foreach (string grammar in GrammarNames.SupportedGrammars)
             {
                 using (Stream stream = ResourceLoader.OpenGrammarPackage(grammar))
-                using (StreamReader reader = new StreamReader(stream))
-                using (JsonTextReader jsonTextReader = new JsonTextReader(reader))
                 {
-                    GrammarDefinition definition = serializer.Deserialize<GrammarDefinition>(jsonTextReader);
+                    GrammarDefinition definition = JsonSerializer.Deserialize<GrammarDefinition>(stream, GrammarDefinitionSerializationContext.Default.GrammarDefinition);
                     _availableGrammars.Add(grammar, definition);
                 }
             }
         }
 
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        void KeepType<
-            [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.All)] T>()
-        {
-            // No-op
-        }
-
         string GetGrammarFile(string scopeName)
         {
             foreach (string grammarName in _availableGrammars.Keys)
@@ -265,4 +243,11 @@ static bool HasGrammar(string id, List<Grammar> grammars)
             return false;
         }
     }
+
+    // Enable Source Generator support for GrammarDefinition
+    [JsonSourceGenerationOptions(GenerationMode = JsonSourceGenerationMode.Metadata)]
+    [JsonSerializable(typeof(GrammarDefinition))]
+    internal sealed partial class GrammarDefinitionSerializationContext : JsonSerializerContext
+    {
+    }
 }

src/TextMateSharp.Grammars/TextMateSharp.Grammars.csproj

@@ -10,14 +10,15 @@
     <Authors>Daniel Peñalba</Authors>
     <Company />
     <PackageLicenseExpression>MIT</PackageLicenseExpression>
+    <EnableTrimAnalyzer>true</EnableTrimAnalyzer>
   </PropertyGroup>
 
   <ItemGroup>
     <EmbeddedResource Include="Resources\Themes\*.json" />
     <EmbeddedResource Include="Resources\Grammars\**\*.json" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="Newtonsoft.Json" Version="$(NewtonsoftJsonVersion)" />
+    <PackageReference Include="System.Text.Json" Version="$(SystemTextJsonVersion)" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\TextMateSharp\TextMateSharp.csproj" />

src/TextMateSharp/Internal/Parser/Json/JSONPListParser.cs

@@ -1,13 +1,12 @@
+using System;
 using System.IO;
-
-using Newtonsoft.Json;
+using System.Text.Json;
 
 namespace TextMateSharp.Internal.Parser.Json
 {
     public class JSONPListParser<T>
     {
-
-        private bool theme;
+        private readonly bool theme;
 
         public JSONPListParser(bool theme)
         {
@@ -18,46 +17,85 @@ public T Parse(StreamReader contents)
         {
             PList<T> pList = new PList<T>(theme);
 
-            JsonReader reader = new JsonTextReader(contents);
+            var buffer = new byte[2048];
+
+            JsonReaderOptions options = new JsonReaderOptions()
+            {
+                AllowTrailingCommas = true,
+                CommentHandling = JsonCommentHandling.Skip
+            };
+
+            int size = contents.BaseStream.Read(buffer, 0, buffer.Length);
+            var reader = new Utf8JsonReader(buffer.AsSpan(0, size), isFinalBlock: false, state: new JsonReaderState(options));
 
             while (true)
             {
-                if (!reader.Read())
+                int bytesRead = -1;
+
+                while (!reader.Read())
+                {
+                    bytesRead = GetMoreBytesFromStream(contents.BaseStream, ref buffer, ref reader);
+
+                    if (bytesRead == 0)
+                        break;
+                }
+
+                if (bytesRead == 0)
                     break;
 
-                JsonToken nextToken = reader.TokenType;
+                var nextToken = reader.TokenType;
                 switch (nextToken)
                 {
-                    case JsonToken.StartArray:
+                    case JsonTokenType.StartArray:
                         pList.StartElement("array");
                         break;
-                    case JsonToken.EndArray:
+                    case JsonTokenType.EndArray:
                         pList.EndElement("array");
                         break;
-                    case JsonToken.StartObject:
+                    case JsonTokenType.StartObject:
                         pList.StartElement("dict");
                         break;
-                    case JsonToken.EndObject:
+                    case JsonTokenType.EndObject:
                         pList.EndElement("dict");
                         break;
-                    case JsonToken.PropertyName:
+                    case JsonTokenType.PropertyName:
                         pList.StartElement("key");
-                        pList.AddString((string)reader.Value);
+                        pList.AddString(reader.GetString());
                         pList.EndElement("key");
                         break;
-                    case JsonToken.String:
+                    case JsonTokenType.String:
                         pList.StartElement("string");
-                        pList.AddString((string)reader.Value);
+                        pList.AddString(reader.GetString());
                         pList.EndElement("string");
                         break;
-                    case JsonToken.Null:
-                    case JsonToken.Boolean:
-                    case JsonToken.Integer:
-                    case JsonToken.Float:
-                        break;
                 }
             }
             return pList.GetResult();
         }
+
+        private static int GetMoreBytesFromStream(Stream stream, ref byte[] buffer, ref Utf8JsonReader reader)
+        {
+            int bytesRead;
+            if (reader.BytesConsumed < buffer.Length)
+            {
+                ReadOnlySpan<byte> leftover = buffer.AsSpan((int)reader.BytesConsumed);
+
+                if (leftover.Length == buffer.Length)
+                {
+                    Array.Resize(ref buffer, buffer.Length * 2);
+                }
+
+                leftover.CopyTo(buffer);
+                bytesRead = stream.Read(buffer, leftover.Length, buffer.Length - leftover.Length);
+                reader = new Utf8JsonReader(buffer.AsSpan(0, bytesRead + leftover.Length), isFinalBlock: bytesRead == 0, reader.CurrentState);
+            }
+            else
+            {
+                bytesRead = stream.Read(buffer, 0, buffer.Length);
+                reader = new Utf8JsonReader(buffer.AsSpan(0, bytesRead), isFinalBlock: bytesRead == 0, reader.CurrentState);
+            }
+
+            return bytesRead;
+        }
     }
 }

src/TextMateSharp/TextMateSharp.csproj

@@ -68,7 +68,7 @@ TextMateSharp uses a wrapper around Oniguruma regex engine. Read below to learn
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Newtonsoft.Json" Version="$(NewtonsoftJsonVersion)" />
+    <PackageReference Include="System.Text.Json" Version="$(SystemTextJsonVersion)" />
   </ItemGroup>
   <Import Project="..\..\build\SourceLink.props" />
   <Import Project="..\..\build\Directory.Build.props" />