Merge branch 'dotnet-dev' into stevenic/dotnet-streaming

microsoft · Dec 9, 2024 · 7cde4d8 · 7cde4d8
2 parents 8669f7a + 1c7d9c9
commit 7cde4d8
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 3 deletions.
diff --git a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Tokenizers/GPTTokenizer.cs b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/AI/Tokenizers/GPTTokenizer.cs
@@ -24,7 +24,7 @@ public class GPTTokenizer : ITokenizer
         /// Creates an instance of `GPTTokenizer`
         /// </summary>
         /// <param name="model">model to encode/decode for</param>
-        public GPTTokenizer(string model) => this._encoding = TiktokenTokenizer.CreateForModel("gpt-4");
+        public GPTTokenizer(string model) => this._encoding = TiktokenTokenizer.CreateForModel(model);
 
         /// <summary>
         /// Encode

diff --git a/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/Microsoft.Teams.AI.csproj b/dotnet/packages/Microsoft.TeamsAI/Microsoft.TeamsAI/Microsoft.Teams.AI.csproj
@@ -45,9 +45,10 @@
     <PackageReference Include="Microsoft.Bot.Builder.Integration.AspNet.Core" Version="4.22.9" />
     <PackageReference Include="Microsoft.Identity.Client" Version="4.61.3" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="Microsoft.ML.Tokenizers" Version="0.22.0-preview.24378.1" />
+    <PackageReference Include="Microsoft.ML.Tokenizers.Data.Cl100kBase" Version="1.0.0" /> <!-- This package is required to support gpt-3.x and gpt-4 models -->
+    <PackageReference Include="Microsoft.ML.Tokenizers.Data.O200kBase" Version="1.0.0" />  <!-- This package is required to support gpt-4o models -->
     <PackageReference Include="OpenAI" Version="2.1.0-beta.1" />
-    <PackageReference Include="System.Text.Json" Version="8.0.5" /> 
+    <PackageReference Include="System.Text.Json" Version="8.0.5" />
   </ItemGroup>
 
   <ItemGroup>