Upgrade tokenizers library to the GA version 1.0.0

microsoft · Dec 2, 2024 · 2e9892a · 2e9892a
1 parent 524f08c
commit 2e9892a
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Tokenizers/GPTTokenizer.cs b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Tokenizers/GPTTokenizer.cs
@@ -24,7 +24,7 @@ public class GPTTokenizer : ITokenizer
         /// Creates an instance of `GPTTokenizer`
         /// </summary>
         /// <param name="model">model to encode/decode for</param>
-        public GPTTokenizer(string model) => this._encoding = TiktokenTokenizer.CreateForModel("gpt-4");
+        public GPTTokenizer(string model) => this._encoding = TiktokenTokenizer.CreateForModel(model);
 
         /// <summary>
         /// Encode

diff --git a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/Microsoft.Teams.AI.csproj b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/Microsoft.Teams.AI.csproj
@@ -45,9 +45,9 @@
     <PackageReference Include="Microsoft.Bot.Builder.Integration.AspNet.Core" Version="4.22.9" />
     <PackageReference Include="Microsoft.Identity.Client" Version="4.61.3" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="Microsoft.ML.Tokenizers" Version="0.22.0-preview.24378.1" />
+    <PackageReference Include="Microsoft.ML.Tokenizers.Data.Cl100kBase" Version="1.0.0" />
     <PackageReference Include="OpenAI" Version="2.1.0-beta.1" />
-    <PackageReference Include="System.Text.Json" Version="8.0.5" /> 
+    <PackageReference Include="System.Text.Json" Version="8.0.5" />
   </ItemGroup>
 
   <ItemGroup>