⬆️ upgrade codetf (#431)

pixee · Jul 25, 2024 · a069684 · a069684
1 parent 964f558
commit a069684
Show file tree

Hide file tree

Showing 9 changed files with 95 additions and 17 deletions.
diff --git a/core-codemods/src/main/java/io/codemodder/codemods/SensitiveDataLoggingCodemod.java b/core-codemods/src/main/java/io/codemodder/codemods/SensitiveDataLoggingCodemod.java
@@ -10,6 +10,7 @@
 import com.github.javaparser.ast.CompilationUnit;
 import com.github.javaparser.ast.stmt.Statement;
 import io.codemodder.*;
+import io.codemodder.codetf.CodeTFAiMetadata;
 import io.codemodder.javaparser.JavaParserChanger;
 import io.codemodder.plugins.llm.OpenAIService;
 import io.codemodder.plugins.llm.StandardModel;
@@ -79,7 +80,9 @@ public CodemodFileScanningResult visit(
         changes.add(change);
       }
     }
-    return CodemodFileScanningResult.from(changes, List.of());
+    return CodemodFileScanningResult.from(
+        // TODO update tokens to be an accurate number
+        changes, List.of(), new CodeTFAiMetadata(service.providerName(), MODEL.id(), 0));
   }
 
   private SensitivityAndFixAnalysis performSensitivityAnalysis(
@@ -100,9 +103,7 @@ private SensitivityAndFixAnalysis performSensitivityAnalysis(
             .formatted(startLine, codeSnippet);
 
     return service.getResponseForPrompt(
-        List.of(new ChatRequestUserMessage(prompt)),
-        StandardModel.GPT_4O_2024_05_13,
-        SensitivityAndFixAnalysisDTO.class);
+        List.of(new ChatRequestUserMessage(prompt)), MODEL, SensitivityAndFixAnalysisDTO.class);
   }
 
   /**
@@ -154,6 +155,8 @@ private static String snippet(final List<String> lines, final int line) {
    */
   private static final int CONTEXT = 10;
 
+  private static final StandardModel MODEL = StandardModel.GPT_4O_2024_05_13;
+
   /** The results of the sensitivity analysis. */
   private interface SensitivityAndFixAnalysis {
 

diff --git a/framework/codemodder-base/build.gradle.kts b/framework/codemodder-base/build.gradle.kts
@@ -21,7 +21,7 @@ dependencies {
     api(libs.java.security.toolkit)
     api(libs.commons.lang3)
 
-    api("io.codemodder:codetf-java:4.0.0")
+    api("io.codemodder:codetf-java:4.1.3")
     api(libs.slf4j.api)
     api(libs.javaparser.core)
     api(libs.javaparser.symbolsolver.core)

diff --git a/framework/codemodder-base/src/main/java/io/codemodder/AICodemodFileScanningResult.java b/framework/codemodder-base/src/main/java/io/codemodder/AICodemodFileScanningResult.java
@@ -0,0 +1,11 @@
+package io.codemodder;
+
+import io.codemodder.codetf.CodeTFAiMetadata;
+import io.codemodder.codetf.UnfixedFinding;
+import java.util.List;
+
+record AICodemodFileScanningResult(
+    List<CodemodChange> changes,
+    List<UnfixedFinding> unfixedFindings,
+    CodeTFAiMetadata codeTFAiMetadata)
+    implements CodemodFileScanningResult, AIMetadataProvider {}
diff --git a/framework/codemodder-base/src/main/java/io/codemodder/AIMetadataProvider.java b/framework/codemodder-base/src/main/java/io/codemodder/AIMetadataProvider.java
@@ -0,0 +1,12 @@
+package io.codemodder;
+
+import io.codemodder.codetf.CodeTFAiMetadata;
+
+/** Marks a class so that it can provide {@link CodeTFAiMetadata} to its consumers. */
+interface AIMetadataProvider {
+
+  /**
+   * @return the {@link CodeTFAiMetadata} describing how a class used an AI service.
+   */
+  CodeTFAiMetadata codeTFAiMetadata();
+}
diff --git a/framework/codemodder-base/src/main/java/io/codemodder/CodemodFileScanningResult.java b/framework/codemodder-base/src/main/java/io/codemodder/CodemodFileScanningResult.java
@@ -1,5 +1,6 @@
 package io.codemodder;
 
+import io.codemodder.codetf.CodeTFAiMetadata;
 import io.codemodder.codetf.UnfixedFinding;
 import java.util.List;
 
@@ -22,6 +23,17 @@ public List<UnfixedFinding> unfixedFindings() {
     };
   }
 
+  /**
+   * Creates a new instance of {@link CodemodFileScanningResult} from the given values, including AI
+   * usage metadata
+   */
+  static CodemodFileScanningResult from(
+      final List<CodemodChange> changes,
+      final List<UnfixedFinding> unfixedFindings,
+      CodeTFAiMetadata codeTFAiMetadata) {
+    return new AICodemodFileScanningResult(changes, unfixedFindings, codeTFAiMetadata);
+  }
+
   /** Creates an empty instance of {@link CodemodFileScanningResult}. */
   static CodemodFileScanningResult none() {
     return from(List.of(), List.of());

diff --git a/framework/codemodder-base/src/main/java/io/codemodder/DefaultCodemodExecutor.java b/framework/codemodder-base/src/main/java/io/codemodder/DefaultCodemodExecutor.java
@@ -157,11 +157,22 @@ public CodeTFResult execute(final List<Path> filePaths) {
 
               // run the codemod on the file
               CodemodFileScanningResult codemodFileScanningResult = codemodRunner.run(context);
+              final CodeTFAiMetadata codeTFAiMetadata;
+              if (codemodFileScanningResult instanceof AIMetadataProvider aiMetadataProvider) {
+                codeTFAiMetadata = aiMetadataProvider.codeTFAiMetadata();
+              } else {
+                codeTFAiMetadata = null;
+              }
               List<CodemodChange> codemodChanges = codemodFileScanningResult.changes();
               if (!codemodChanges.isEmpty()) {
                 synchronized (this) {
                   FilesUpdateResult updateResult =
-                      updateFiles(codeChanger, filePath, beforeFileContents, codemodChanges);
+                      updateFiles(
+                          codeChanger,
+                          filePath,
+                          beforeFileContents,
+                          codemodChanges,
+                          codeTFAiMetadata);
                   unscannableFiles.addAll(updateResult.filesFailedToChange());
                   changeset.addAll(updateResult.changeset());
                 }
@@ -201,6 +212,8 @@ public CodeTFResult execute(final List<Path> filePaths) {
             codeChanger.getSummary(),
             codeChanger.getDescription(),
             detectionTool,
+            // TODO augment CodeTF with failure metadata
+            null,
             unscannableFiles.stream()
                 .map(file -> getRelativePath(projectDir, file))
                 .collect(Collectors.toSet()),
@@ -223,7 +236,8 @@ private FilesUpdateResult updateFiles(
       final CodeChanger codeChanger,
       final Path filePath,
       final String beforeFileContents,
-      final List<CodemodChange> codemodChanges)
+      final List<CodemodChange> codemodChanges,
+      final CodeTFAiMetadata codeTFAiMetadata)
       throws IOException {
 
     List<Path> filesFailedToChange = List.of();
@@ -272,7 +286,9 @@ private FilesUpdateResult updateFiles(
 
     // create a changeset for this file change + its downstream dependency changes
     List<CodeTFChangesetEntry> changeset = new ArrayList<>();
-    changeset.add(new CodeTFChangesetEntry(getRelativePath(projectDir, filePath), diff, changes));
+    changeset.add(
+        new CodeTFChangesetEntry(
+            getRelativePath(projectDir, filePath), diff, changes, codeTFAiMetadata));
     changeset.addAll(dependencyChangesetEntries);
 
     // update the cache

diff --git a/plugins/codemodder-plugin-llm/src/main/java/io/codemodder/plugins/llm/OpenAIService.java b/plugins/codemodder-plugin-llm/src/main/java/io/codemodder/plugins/llm/OpenAIService.java
@@ -23,6 +23,8 @@ public class OpenAIService {
   private final ModelMapper modelMapper;
   private boolean serviceAvailable = true;
 
+  private final String providerName;
+
   private static OpenAIClientBuilder builder(final KeyCredential key) {
     HttpClientOptions clientOptions = new HttpClientOptions();
     clientOptions.setReadTimeout(Duration.ofSeconds(TIMEOUT_SECONDS));
@@ -32,20 +34,27 @@ private static OpenAIClientBuilder builder(final KeyCredential key) {
         .credential(key);
   }
 
-  OpenAIService(final boolean serviceAvailable) {
-    this.serviceAvailable = serviceAvailable;
+  OpenAIService() {
+    this.serviceAvailable = false;
     this.modelMapper = null;
     this.api = null;
+    this.providerName = null;
   }
 
-  OpenAIService(final ModelMapper mapper, final KeyCredential key) {
+  OpenAIService(final ModelMapper mapper, final KeyCredential key, final String providerName) {
     this.modelMapper = mapper;
     this.api = builder(key).buildClient();
+    this.providerName = providerName;
   }
 
-  OpenAIService(final ModelMapper mapper, final KeyCredential key, final String endpoint) {
+  OpenAIService(
+      final ModelMapper mapper,
+      final KeyCredential key,
+      final String endpoint,
+      final String providerName) {
     this.modelMapper = mapper;
     this.api = builder(key).endpoint(endpoint).buildClient();
+    this.providerName = providerName;
   }
 
   /**
@@ -56,7 +65,9 @@ private static OpenAIClientBuilder builder(final KeyCredential key) {
    */
   public static OpenAIService fromOpenAI(final String token) {
     return new OpenAIService(
-        new EnvironmentBasedModelMapper(), new KeyCredential(Objects.requireNonNull(token)));
+        new EnvironmentBasedModelMapper(),
+        new KeyCredential(Objects.requireNonNull(token)),
+        "openai");
   }
 
   /**
@@ -70,11 +81,12 @@ public static OpenAIService fromAzureOpenAI(final String token, final String end
     return new OpenAIService(
         new EnvironmentBasedModelMapper(),
         new AzureKeyCredential(Objects.requireNonNull(token)),
-        Objects.requireNonNull(endpoint));
+        Objects.requireNonNull(endpoint),
+        "azure-openai");
   }
 
   public static OpenAIService noServiceAvailable() {
-    return new OpenAIService(false);
+    return new OpenAIService();
   }
 
   /**
@@ -86,6 +98,10 @@ public boolean isServiceAvailable() {
     return serviceAvailable;
   }
 
+  public String providerName() {
+    return providerName;
+  }
+
   /**
    * Gets the completion for the given messages.
    *

diff --git a/.../main/java/io/codemodder/plugins/llm/SarifToLLMForBinaryVerificationAndFixingCodemod.java b/.../main/java/io/codemodder/plugins/llm/SarifToLLMForBinaryVerificationAndFixingCodemod.java
@@ -9,6 +9,7 @@
 import com.github.difflib.DiffUtils;
 import com.github.difflib.patch.Patch;
 import io.codemodder.*;
+import io.codemodder.codetf.CodeTFAiMetadata;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.nio.file.Files;
@@ -106,7 +107,9 @@ public CodemodFileScanningResult onFileFound(
       // Report all the changes at the line number of the first change.
       int line = patch.getDeltas().get(0).getSource().getPosition() + 1; // Position is 0-based.
       List<CodemodChange> changes = List.of(CodemodChange.from(line, fix.getFixDescription()));
-      return CodemodFileScanningResult.withOnlyChanges(changes);
+      // TODO update tokens to be an accurate number
+      return CodemodFileScanningResult.from(
+          changes, List.of(), new CodeTFAiMetadata(openAI.providerName(), model.id(), 0));
     } catch (IOException e) {
       logger.error("failed to process: {}", context.path(), e);
       throw new UncheckedIOException(e);

diff --git a/...-plugin-llm/src/main/java/io/codemodder/plugins/llm/SarifToLLMForMultiOutcomeCodemod.java b/...-plugin-llm/src/main/java/io/codemodder/plugins/llm/SarifToLLMForMultiOutcomeCodemod.java
@@ -10,6 +10,7 @@
 import com.github.difflib.DiffUtils;
 import com.github.difflib.patch.Patch;
 import io.codemodder.*;
+import io.codemodder.codetf.CodeTFAiMetadata;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.nio.file.Files;
@@ -83,7 +84,11 @@ public CodemodFileScanningResult onFileFound(
       Optional<CodemodChange> change = processResult(context, result);
       change.ifPresent(changes::add);
     }
-    return CodemodFileScanningResult.withOnlyChanges(List.copyOf(changes));
+    return CodemodFileScanningResult.from(
+        List.copyOf(changes),
+        List.of(),
+        // TODO use the actual token count
+        new CodeTFAiMetadata(openAI.providerName(), codeChangingModel.id(), 0));
   }
 
   private Optional<CodemodChange> processResult(