⬆️ Use Correct Encoding Type for GPT-4o (#441)

Upgraded jtokkit to a newer version that supports the encoding type we need.
pixee · Aug 21, 2024 · eb9084a · eb9084a
1 parent 9170c87
commit eb9084a
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 6 deletions.
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -6,7 +6,7 @@ javaparser-symbolsolver = "3.15.15"
 java-security-toolkit = "1.2.0"
 java-security-toolkit-xstream = "1.0.2"
 javax-inject = "1"
-jtokkit = "0.6.1"
+jtokkit = "1.1.0"
 commons-jexl = "3.2.1"
 logback = "1.4.5"
 maven = "3.8.7"

diff --git a/plugins/codemodder-plugin-llm/src/main/java/io/codemodder/plugins/llm/StandardModel.java b/plugins/codemodder-plugin-llm/src/main/java/io/codemodder/plugins/llm/StandardModel.java
@@ -24,13 +24,9 @@ public int tokens(final List<String> messages) {
     }
   },
   GPT_4O_2024_05_13("gpt-4o-2024-05-13", 128_000) {
-    /**
-     * This is wrong - we copy / pasted from GPT 3.5 while we await GPT-4o token counting support <a
-     * href="https://github.com/knuddelsgmbh/jtokkit/issues/96">from upstream utility</a>.
-     */
     @Override
     public int tokens(final List<String> messages) {
-      return Tokens.countTokens(messages, 3, EncodingType.CL100K_BASE);
+      return Tokens.countTokens(messages, 3, EncodingType.O200K_BASE);
     }
   };