diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 2ecbde784..d5c3bd51e 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -6,7 +6,7 @@ javaparser-symbolsolver = "3.15.15" java-security-toolkit = "1.2.0" java-security-toolkit-xstream = "1.0.2" javax-inject = "1" -jtokkit = "0.6.1" +jtokkit = "1.1.0" commons-jexl = "3.2.1" logback = "1.4.5" maven = "3.8.7" diff --git a/plugins/codemodder-plugin-llm/src/main/java/io/codemodder/plugins/llm/StandardModel.java b/plugins/codemodder-plugin-llm/src/main/java/io/codemodder/plugins/llm/StandardModel.java index ab586fb44..aee5b8701 100644 --- a/plugins/codemodder-plugin-llm/src/main/java/io/codemodder/plugins/llm/StandardModel.java +++ b/plugins/codemodder-plugin-llm/src/main/java/io/codemodder/plugins/llm/StandardModel.java @@ -24,13 +24,9 @@ public int tokens(final List messages) { } }, GPT_4O_2024_05_13("gpt-4o-2024-05-13", 128_000) { - /** - * This is wrong - we copy / pasted from GPT 3.5 while we await GPT-4o token counting support from upstream utility. - */ @Override public int tokens(final List messages) { - return Tokens.countTokens(messages, 3, EncodingType.CL100K_BASE); + return Tokens.countTokens(messages, 3, EncodingType.O200K_BASE); } };