From 2e9892aec818c84acd785dcf8aa4cf70b5547575 Mon Sep 17 00:00:00 2001
From: Tarek Mahmoud Sayed <tarekms@microsoft.com>
Date: Mon, 2 Dec 2024 10:06:52 -0800
Subject: [PATCH] Upgrade tokenizers library to the GA version 1.0.0

---
 .../Microsoft.TeamsAI/AI/Tokenizers/GPTTokenizer.cs           | 2 +-
 .../Microsoft.TeamsAI/Microsoft.Teams.AI.csproj               | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Tokenizers/GPTTokenizer.cs b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Tokenizers/GPTTokenizer.cs
index 72b6e52cd..ab140f8b8 100644
--- a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Tokenizers/GPTTokenizer.cs
+++ b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Tokenizers/GPTTokenizer.cs
@@ -24,7 +24,7 @@ public class GPTTokenizer : ITokenizer
         /// Creates an instance of `GPTTokenizer`
         /// </summary>
         /// <param name="model">model to encode/decode for</param>
-        public GPTTokenizer(string model) => this._encoding = TiktokenTokenizer.CreateForModel("gpt-4");
+        public GPTTokenizer(string model) => this._encoding = TiktokenTokenizer.CreateForModel(model);
 
         /// <summary>
         /// Encode
diff --git a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/Microsoft.Teams.AI.csproj b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/Microsoft.Teams.AI.csproj
index 5550baf0c..d8f4dbba4 100644
--- a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/Microsoft.Teams.AI.csproj
+++ b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/Microsoft.Teams.AI.csproj
@@ -45,9 +45,9 @@
     <PackageReference Include="Microsoft.Bot.Builder.Integration.AspNet.Core" Version="4.22.9" />
     <PackageReference Include="Microsoft.Identity.Client" Version="4.61.3" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="Microsoft.ML.Tokenizers" Version="0.22.0-preview.24378.1" />
+    <PackageReference Include="Microsoft.ML.Tokenizers.Data.Cl100kBase" Version="1.0.0" />
     <PackageReference Include="OpenAI" Version="2.1.0-beta.1" />
-    <PackageReference Include="System.Text.Json" Version="8.0.5" /> 
+    <PackageReference Include="System.Text.Json" Version="8.0.5" />
   </ItemGroup>
 
   <ItemGroup>