From a069684da532d6078ce6c5559da24dd27945a070 Mon Sep 17 00:00:00 2001 From: Ryan Dens Date: Thu, 25 Jul 2024 13:44:55 -0700 Subject: [PATCH] :arrow_up: upgrade codetf (#431) --- .../codemods/SensitiveDataLoggingCodemod.java | 11 ++++--- framework/codemodder-base/build.gradle.kts | 2 +- .../AICodemodFileScanningResult.java | 11 +++++++ .../io/codemodder/AIMetadataProvider.java | 12 ++++++++ .../codemodder/CodemodFileScanningResult.java | 12 ++++++++ .../io/codemodder/DefaultCodemodExecutor.java | 22 ++++++++++++-- .../codemodder/plugins/llm/OpenAIService.java | 30 ++++++++++++++----- ...ForBinaryVerificationAndFixingCodemod.java | 5 +++- .../llm/SarifToLLMForMultiOutcomeCodemod.java | 7 ++++- 9 files changed, 95 insertions(+), 17 deletions(-) create mode 100644 framework/codemodder-base/src/main/java/io/codemodder/AICodemodFileScanningResult.java create mode 100644 framework/codemodder-base/src/main/java/io/codemodder/AIMetadataProvider.java diff --git a/core-codemods/src/main/java/io/codemodder/codemods/SensitiveDataLoggingCodemod.java b/core-codemods/src/main/java/io/codemodder/codemods/SensitiveDataLoggingCodemod.java index 6705b00ea..e4b3bb963 100644 --- a/core-codemods/src/main/java/io/codemodder/codemods/SensitiveDataLoggingCodemod.java +++ b/core-codemods/src/main/java/io/codemodder/codemods/SensitiveDataLoggingCodemod.java @@ -10,6 +10,7 @@ import com.github.javaparser.ast.CompilationUnit; import com.github.javaparser.ast.stmt.Statement; import io.codemodder.*; +import io.codemodder.codetf.CodeTFAiMetadata; import io.codemodder.javaparser.JavaParserChanger; import io.codemodder.plugins.llm.OpenAIService; import io.codemodder.plugins.llm.StandardModel; @@ -79,7 +80,9 @@ public CodemodFileScanningResult visit( changes.add(change); } } - return CodemodFileScanningResult.from(changes, List.of()); + return CodemodFileScanningResult.from( + // TODO update tokens to be an accurate number + changes, List.of(), new CodeTFAiMetadata(service.providerName(), MODEL.id(), 0)); } private SensitivityAndFixAnalysis performSensitivityAnalysis( @@ -100,9 +103,7 @@ private SensitivityAndFixAnalysis performSensitivityAnalysis( .formatted(startLine, codeSnippet); return service.getResponseForPrompt( - List.of(new ChatRequestUserMessage(prompt)), - StandardModel.GPT_4O_2024_05_13, - SensitivityAndFixAnalysisDTO.class); + List.of(new ChatRequestUserMessage(prompt)), MODEL, SensitivityAndFixAnalysisDTO.class); } /** @@ -154,6 +155,8 @@ private static String snippet(final List lines, final int line) { */ private static final int CONTEXT = 10; + private static final StandardModel MODEL = StandardModel.GPT_4O_2024_05_13; + /** The results of the sensitivity analysis. */ private interface SensitivityAndFixAnalysis { diff --git a/framework/codemodder-base/build.gradle.kts b/framework/codemodder-base/build.gradle.kts index 9d6bfc718..c3cf3581b 100644 --- a/framework/codemodder-base/build.gradle.kts +++ b/framework/codemodder-base/build.gradle.kts @@ -21,7 +21,7 @@ dependencies { api(libs.java.security.toolkit) api(libs.commons.lang3) - api("io.codemodder:codetf-java:4.0.0") + api("io.codemodder:codetf-java:4.1.3") api(libs.slf4j.api) api(libs.javaparser.core) api(libs.javaparser.symbolsolver.core) diff --git a/framework/codemodder-base/src/main/java/io/codemodder/AICodemodFileScanningResult.java b/framework/codemodder-base/src/main/java/io/codemodder/AICodemodFileScanningResult.java new file mode 100644 index 000000000..bf8fed6de --- /dev/null +++ b/framework/codemodder-base/src/main/java/io/codemodder/AICodemodFileScanningResult.java @@ -0,0 +1,11 @@ +package io.codemodder; + +import io.codemodder.codetf.CodeTFAiMetadata; +import io.codemodder.codetf.UnfixedFinding; +import java.util.List; + +record AICodemodFileScanningResult( + List changes, + List unfixedFindings, + CodeTFAiMetadata codeTFAiMetadata) + implements CodemodFileScanningResult, AIMetadataProvider {} diff --git a/framework/codemodder-base/src/main/java/io/codemodder/AIMetadataProvider.java b/framework/codemodder-base/src/main/java/io/codemodder/AIMetadataProvider.java new file mode 100644 index 000000000..abaf539dd --- /dev/null +++ b/framework/codemodder-base/src/main/java/io/codemodder/AIMetadataProvider.java @@ -0,0 +1,12 @@ +package io.codemodder; + +import io.codemodder.codetf.CodeTFAiMetadata; + +/** Marks a class so that it can provide {@link CodeTFAiMetadata} to its consumers. */ +interface AIMetadataProvider { + + /** + * @return the {@link CodeTFAiMetadata} describing how a class used an AI service. + */ + CodeTFAiMetadata codeTFAiMetadata(); +} diff --git a/framework/codemodder-base/src/main/java/io/codemodder/CodemodFileScanningResult.java b/framework/codemodder-base/src/main/java/io/codemodder/CodemodFileScanningResult.java index 834ab3f5d..a701e1e10 100644 --- a/framework/codemodder-base/src/main/java/io/codemodder/CodemodFileScanningResult.java +++ b/framework/codemodder-base/src/main/java/io/codemodder/CodemodFileScanningResult.java @@ -1,5 +1,6 @@ package io.codemodder; +import io.codemodder.codetf.CodeTFAiMetadata; import io.codemodder.codetf.UnfixedFinding; import java.util.List; @@ -22,6 +23,17 @@ public List unfixedFindings() { }; } + /** + * Creates a new instance of {@link CodemodFileScanningResult} from the given values, including AI + * usage metadata + */ + static CodemodFileScanningResult from( + final List changes, + final List unfixedFindings, + CodeTFAiMetadata codeTFAiMetadata) { + return new AICodemodFileScanningResult(changes, unfixedFindings, codeTFAiMetadata); + } + /** Creates an empty instance of {@link CodemodFileScanningResult}. */ static CodemodFileScanningResult none() { return from(List.of(), List.of()); diff --git a/framework/codemodder-base/src/main/java/io/codemodder/DefaultCodemodExecutor.java b/framework/codemodder-base/src/main/java/io/codemodder/DefaultCodemodExecutor.java index 53f8bbc3e..f327d630a 100644 --- a/framework/codemodder-base/src/main/java/io/codemodder/DefaultCodemodExecutor.java +++ b/framework/codemodder-base/src/main/java/io/codemodder/DefaultCodemodExecutor.java @@ -157,11 +157,22 @@ public CodeTFResult execute(final List filePaths) { // run the codemod on the file CodemodFileScanningResult codemodFileScanningResult = codemodRunner.run(context); + final CodeTFAiMetadata codeTFAiMetadata; + if (codemodFileScanningResult instanceof AIMetadataProvider aiMetadataProvider) { + codeTFAiMetadata = aiMetadataProvider.codeTFAiMetadata(); + } else { + codeTFAiMetadata = null; + } List codemodChanges = codemodFileScanningResult.changes(); if (!codemodChanges.isEmpty()) { synchronized (this) { FilesUpdateResult updateResult = - updateFiles(codeChanger, filePath, beforeFileContents, codemodChanges); + updateFiles( + codeChanger, + filePath, + beforeFileContents, + codemodChanges, + codeTFAiMetadata); unscannableFiles.addAll(updateResult.filesFailedToChange()); changeset.addAll(updateResult.changeset()); } @@ -201,6 +212,8 @@ public CodeTFResult execute(final List filePaths) { codeChanger.getSummary(), codeChanger.getDescription(), detectionTool, + // TODO augment CodeTF with failure metadata + null, unscannableFiles.stream() .map(file -> getRelativePath(projectDir, file)) .collect(Collectors.toSet()), @@ -223,7 +236,8 @@ private FilesUpdateResult updateFiles( final CodeChanger codeChanger, final Path filePath, final String beforeFileContents, - final List codemodChanges) + final List codemodChanges, + final CodeTFAiMetadata codeTFAiMetadata) throws IOException { List filesFailedToChange = List.of(); @@ -272,7 +286,9 @@ private FilesUpdateResult updateFiles( // create a changeset for this file change + its downstream dependency changes List changeset = new ArrayList<>(); - changeset.add(new CodeTFChangesetEntry(getRelativePath(projectDir, filePath), diff, changes)); + changeset.add( + new CodeTFChangesetEntry( + getRelativePath(projectDir, filePath), diff, changes, codeTFAiMetadata)); changeset.addAll(dependencyChangesetEntries); // update the cache diff --git a/plugins/codemodder-plugin-llm/src/main/java/io/codemodder/plugins/llm/OpenAIService.java b/plugins/codemodder-plugin-llm/src/main/java/io/codemodder/plugins/llm/OpenAIService.java index 0fe4d63d8..81415962d 100644 --- a/plugins/codemodder-plugin-llm/src/main/java/io/codemodder/plugins/llm/OpenAIService.java +++ b/plugins/codemodder-plugin-llm/src/main/java/io/codemodder/plugins/llm/OpenAIService.java @@ -23,6 +23,8 @@ public class OpenAIService { private final ModelMapper modelMapper; private boolean serviceAvailable = true; + private final String providerName; + private static OpenAIClientBuilder builder(final KeyCredential key) { HttpClientOptions clientOptions = new HttpClientOptions(); clientOptions.setReadTimeout(Duration.ofSeconds(TIMEOUT_SECONDS)); @@ -32,20 +34,27 @@ private static OpenAIClientBuilder builder(final KeyCredential key) { .credential(key); } - OpenAIService(final boolean serviceAvailable) { - this.serviceAvailable = serviceAvailable; + OpenAIService() { + this.serviceAvailable = false; this.modelMapper = null; this.api = null; + this.providerName = null; } - OpenAIService(final ModelMapper mapper, final KeyCredential key) { + OpenAIService(final ModelMapper mapper, final KeyCredential key, final String providerName) { this.modelMapper = mapper; this.api = builder(key).buildClient(); + this.providerName = providerName; } - OpenAIService(final ModelMapper mapper, final KeyCredential key, final String endpoint) { + OpenAIService( + final ModelMapper mapper, + final KeyCredential key, + final String endpoint, + final String providerName) { this.modelMapper = mapper; this.api = builder(key).endpoint(endpoint).buildClient(); + this.providerName = providerName; } /** @@ -56,7 +65,9 @@ private static OpenAIClientBuilder builder(final KeyCredential key) { */ public static OpenAIService fromOpenAI(final String token) { return new OpenAIService( - new EnvironmentBasedModelMapper(), new KeyCredential(Objects.requireNonNull(token))); + new EnvironmentBasedModelMapper(), + new KeyCredential(Objects.requireNonNull(token)), + "openai"); } /** @@ -70,11 +81,12 @@ public static OpenAIService fromAzureOpenAI(final String token, final String end return new OpenAIService( new EnvironmentBasedModelMapper(), new AzureKeyCredential(Objects.requireNonNull(token)), - Objects.requireNonNull(endpoint)); + Objects.requireNonNull(endpoint), + "azure-openai"); } public static OpenAIService noServiceAvailable() { - return new OpenAIService(false); + return new OpenAIService(); } /** @@ -86,6 +98,10 @@ public boolean isServiceAvailable() { return serviceAvailable; } + public String providerName() { + return providerName; + } + /** * Gets the completion for the given messages. * diff --git a/plugins/codemodder-plugin-llm/src/main/java/io/codemodder/plugins/llm/SarifToLLMForBinaryVerificationAndFixingCodemod.java b/plugins/codemodder-plugin-llm/src/main/java/io/codemodder/plugins/llm/SarifToLLMForBinaryVerificationAndFixingCodemod.java index cda978cc2..58b3e9973 100644 --- a/plugins/codemodder-plugin-llm/src/main/java/io/codemodder/plugins/llm/SarifToLLMForBinaryVerificationAndFixingCodemod.java +++ b/plugins/codemodder-plugin-llm/src/main/java/io/codemodder/plugins/llm/SarifToLLMForBinaryVerificationAndFixingCodemod.java @@ -9,6 +9,7 @@ import com.github.difflib.DiffUtils; import com.github.difflib.patch.Patch; import io.codemodder.*; +import io.codemodder.codetf.CodeTFAiMetadata; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.file.Files; @@ -106,7 +107,9 @@ public CodemodFileScanningResult onFileFound( // Report all the changes at the line number of the first change. int line = patch.getDeltas().get(0).getSource().getPosition() + 1; // Position is 0-based. List changes = List.of(CodemodChange.from(line, fix.getFixDescription())); - return CodemodFileScanningResult.withOnlyChanges(changes); + // TODO update tokens to be an accurate number + return CodemodFileScanningResult.from( + changes, List.of(), new CodeTFAiMetadata(openAI.providerName(), model.id(), 0)); } catch (IOException e) { logger.error("failed to process: {}", context.path(), e); throw new UncheckedIOException(e); diff --git a/plugins/codemodder-plugin-llm/src/main/java/io/codemodder/plugins/llm/SarifToLLMForMultiOutcomeCodemod.java b/plugins/codemodder-plugin-llm/src/main/java/io/codemodder/plugins/llm/SarifToLLMForMultiOutcomeCodemod.java index 26168b01a..a5d06cea8 100644 --- a/plugins/codemodder-plugin-llm/src/main/java/io/codemodder/plugins/llm/SarifToLLMForMultiOutcomeCodemod.java +++ b/plugins/codemodder-plugin-llm/src/main/java/io/codemodder/plugins/llm/SarifToLLMForMultiOutcomeCodemod.java @@ -10,6 +10,7 @@ import com.github.difflib.DiffUtils; import com.github.difflib.patch.Patch; import io.codemodder.*; +import io.codemodder.codetf.CodeTFAiMetadata; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.file.Files; @@ -83,7 +84,11 @@ public CodemodFileScanningResult onFileFound( Optional change = processResult(context, result); change.ifPresent(changes::add); } - return CodemodFileScanningResult.withOnlyChanges(List.copyOf(changes)); + return CodemodFileScanningResult.from( + List.copyOf(changes), + List.of(), + // TODO use the actual token count + new CodeTFAiMetadata(openAI.providerName(), codeChangingModel.id(), 0)); } private Optional processResult(