Add Tiktoken's missing model names (#7111)

dotnet · Mar 29, 2024 · c96aac7 · c96aac7
1 parent 01e6dd2
commit c96aac7
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 2 deletions.
diff --git a/src/Microsoft.ML.Tokenizers/Model/Tiktoken.cs b/src/Microsoft.ML.Tokenizers/Model/Tiktoken.cs
@@ -676,8 +676,9 @@ internal enum ModelEncoding
         private static readonly (string Prefix, ModelEncoding Encoding)[] _modelPrefixToEncoding =
                                                             [
                                                                 // chat
-                                                                ("gpt-4-", ModelEncoding.Cl100kBase),  // e.g., gpt-4-0314, etc., plus gpt-4-32k
-                                                                ("gpt-3.5-turbo-", ModelEncoding.Cl100kBase) // e.g, gpt-3.5-turbo-0301, -0401, etc.
+                                                                ( "gpt-4-", ModelEncoding.Cl100kBase),          // e.g., gpt-4-0314, etc., plus gpt-4-32k
+                                                                ( "gpt-3.5-turbo-", ModelEncoding.Cl100kBase),  // e.g, gpt-3.5-turbo-0301, -0401, etc.
+                                                                ( "gpt-35-turbo-", ModelEncoding.Cl100kBase )   // Azure deployment name
                                                             ];
 
         private static readonly Dictionary<string, ModelEncoding> _modelToEncoding =
@@ -686,6 +687,9 @@ internal enum ModelEncoding
                                                                 // chat
                                                                 { "gpt-4", ModelEncoding.Cl100kBase },
                                                                 { "gpt-3.5-turbo", ModelEncoding.Cl100kBase },
+                                                                { "gpt-3.5-turbo-16k", ModelEncoding.Cl100kBase },
+                                                                { "gpt-35-turbo", ModelEncoding.Cl100kBase },     // Azure deployment name
+                                                                { "gpt-35-turbo-16k", ModelEncoding.Cl100kBase }, // Azure deployment name
 
                                                                 // text
                                                                 { "text-davinci-003", ModelEncoding.P50kBase },

diff --git a/test/Microsoft.ML.Tokenizers.Tests/TitokenTests.cs b/test/Microsoft.ML.Tokenizers.Tests/TitokenTests.cs
@@ -312,6 +312,10 @@ public void TestEncodeR50kBase()
         [InlineData("gpt-4-")]
         [InlineData("gpt-3.5-turbo")]
         [InlineData("gpt-3.5-turbo-")]
+        [InlineData("gpt-3.5-turbo-16k")]
+        [InlineData("gpt-35-turbo")]
+        [InlineData("gpt-35-turbo-16k")]
+        [InlineData("gpt-35-turbo-")]
         [InlineData("text-davinci-003")]
         [InlineData("text-davinci-002")]
         [InlineData("text-davinci-001")]