From eb9084a01d90beb15d3f2b577a835799654c9775 Mon Sep 17 00:00:00 2001
From: Johnathan Gilday <johnathan.gilday@pixee.ai>
Date: Wed, 21 Aug 2024 11:07:56 -0400
Subject: [PATCH] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Use=20Correct=20Encoding?=
 =?UTF-8?q?=20Type=20for=20GPT-4o=20(#441)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Upgraded jtokkit to a newer version that supports the encoding type we
need.
---
 gradle/libs.versions.toml                                   | 2 +-
 .../main/java/io/codemodder/plugins/llm/StandardModel.java  | 6 +-----
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index 2ecbde784..d5c3bd51e 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -6,7 +6,7 @@ javaparser-symbolsolver = "3.15.15"
 java-security-toolkit = "1.2.0"
 java-security-toolkit-xstream = "1.0.2"
 javax-inject = "1"
-jtokkit = "0.6.1"
+jtokkit = "1.1.0"
 commons-jexl = "3.2.1"
 logback = "1.4.5"
 maven = "3.8.7"
diff --git a/plugins/codemodder-plugin-llm/src/main/java/io/codemodder/plugins/llm/StandardModel.java b/plugins/codemodder-plugin-llm/src/main/java/io/codemodder/plugins/llm/StandardModel.java
index ab586fb44..aee5b8701 100644
--- a/plugins/codemodder-plugin-llm/src/main/java/io/codemodder/plugins/llm/StandardModel.java
+++ b/plugins/codemodder-plugin-llm/src/main/java/io/codemodder/plugins/llm/StandardModel.java
@@ -24,13 +24,9 @@ public int tokens(final List<String> messages) {
     }
   },
   GPT_4O_2024_05_13("gpt-4o-2024-05-13", 128_000) {
-    /**
-     * This is wrong - we copy / pasted from GPT 3.5 while we await GPT-4o token counting support <a
-     * href="https://github.com/knuddelsgmbh/jtokkit/issues/96">from upstream utility</a>.
-     */
     @Override
     public int tokens(final List<String> messages) {
-      return Tokens.countTokens(messages, 3, EncodingType.CL100K_BASE);
+      return Tokens.countTokens(messages, 3, EncodingType.O200K_BASE);
     }
   };