Update Library Model Loader to mark fields Nullable (#220)

Followup on an update in NullAway recent change [#878](uber/NullAway#878) which enables NullAway to mark fields as `@Nullable`. This PR updates our `LibraryModelsLoader` to enable communication of Annotator and NullAway to mark fields `@Nullable` while analyzing downstream dependencies. This is required to prepare the code for the upcoming change which enables Annotator to be informed of impacts of making fields `@Nullable` on downstream dependencies.

annotator-core/build.gradle

@@ -57,7 +57,7 @@ googleJavaFormat {
 }
 
 // Should be the latest supporting version of NullAway.
-def NULLAWAY_TEST = "0.10.10"
+def NULLAWAY_TEST = "0.10.19"
 
 tasks.test.dependsOn(':annotator-scanner:publishToMavenLocal')
 

annotator-core/src/main/java/edu/ucr/cs/riple/core/Config.java

@@ -105,8 +105,9 @@ public class Config {
   /** Sets of context path information for all downstream dependencies. */
   public final ImmutableSet<ModuleConfiguration> downstreamConfigurations;
   /**
-   * Path to NullAway library model loader, which enables the communication between annotator and
-   * NullAway when processing downstream dependencies.
+   * Path to NullAway library model loader resource directory, which enables the communication
+   * between annotator and NullAway when processing downstream dependencies. Annotator will write
+   * annotation on files in this directory and the using Library Model Loader reads them.
    */
   public final Path nullawayLibraryModelLoaderPath;
   /** Command to build the all downstream dependencies at once. */
@@ -299,7 +300,7 @@ public Config(String[] args) {
             "nlmlp",
             "nullaway-library-model-loader-path",
             true,
-            "NullAway Library Model loader path");
+            "NullAway Library Model loader resource directory path");
     nullawayLibraryModelLoaderPathOption.setRequired(false);
     options.addOption(nullawayLibraryModelLoaderPathOption);
     // Down stream analysis: Analysis mode.

annotator-core/src/main/java/edu/ucr/cs/riple/core/injectors/VirtualInjector.java

@@ -24,6 +24,7 @@
 
 package edu.ucr.cs.riple.core.injectors;
 
+import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
 import edu.ucr.cs.riple.core.Config;
 import edu.ucr.cs.riple.core.Context;
@@ -33,9 +34,10 @@
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.charset.Charset;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Set;
-import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * Wrapper tool used to inject annotations virtually to the source code. This injector serializes
@@ -44,21 +46,32 @@
  */
 public class VirtualInjector extends AnnotationInjector {
 
-  /** Path to library model loader */
-  private final Path libraryModelPath;
+  /** Path to library model loader resources directory */
+  private final Path libraryModelResourcesDirectoryPath;
   /**
    * Annotator configuration, required to check if downstream dependencies analysis is activated or
    * retrieve the path to library model loader.
    */
   private final Config config;
+  /** Name of the resource file in library model loader which contains list of nullable methods. */
+  public static final String NULLABLE_METHOD_LIST_FILE_NAME = "nullable-methods.tsv";
+  /** Name of the resource file in library model loader which contains list of nullable fields. */
+  public static final String NULLABLE_FIELD_LIST_FILE_NAME = "nullable-fields.tsv";
 
   public VirtualInjector(Context context) {
     super(context);
     this.config = context.config;
-    this.libraryModelPath = config.nullawayLibraryModelLoaderPath;
+    this.libraryModelResourcesDirectoryPath = config.nullawayLibraryModelLoaderPath;
     if (config.downStreamDependenciesAnalysisActivated) {
+      try {
+        // make the directories for resources
+        Files.createDirectories(libraryModelResourcesDirectoryPath);
+      } catch (IOException e) {
+        throw new RuntimeException(
+            "Error happened for creating directory: " + libraryModelResourcesDirectoryPath, e);
+      }
       Preconditions.checkNotNull(
-          libraryModelPath,
+          libraryModelResourcesDirectoryPath,
           "NullawayLibraryModelLoaderPath cannot be null while downstream dependencies analysis is activated.");
       clear();
     }
@@ -75,33 +88,69 @@ public void injectAnnotations(Set<AddAnnotation> changes) {
       throw new IllegalStateException(
           "Downstream dependencies analysis not activated, cannot inject annotations virtually!");
     }
-    try (BufferedOutputStream os =
-        new BufferedOutputStream(new FileOutputStream(libraryModelPath.toFile()))) {
-      Set<String> rows =
-          changes.stream()
-              .filter(addAnnotation -> addAnnotation.getLocation().isOnMethod())
-              .map(
-                  annot ->
-                      annot.getLocation().clazz
-                          + "\t"
-                          + annot.getLocation().toMethod().method
-                          + "\n")
-              .collect(Collectors.toSet());
-      for (String row : rows) {
-        os.write(row.getBytes(Charset.defaultCharset()), 0, row.length());
-      }
-      os.flush();
+    // write methods
+    writeAnnotationsToFile(
+        changes.stream().filter(addAnnotation -> addAnnotation.getLocation().isOnMethod()),
+        libraryModelResourcesDirectoryPath.resolve(NULLABLE_METHOD_LIST_FILE_NAME),
+        annot ->
+            Stream.of(
+                annot.getLocation().clazz + "\t" + annot.getLocation().toMethod().method + "\n"));
+    // write fields
+    writeAnnotationsToFile(
+        changes.stream().filter(addAnnotation -> addAnnotation.getLocation().isOnField()),
+        libraryModelResourcesDirectoryPath.resolve(NULLABLE_FIELD_LIST_FILE_NAME),
+        annot ->
+            // An annotation on a single statement with multiple declaration will be considered for
+            // all declared variables. Hence, we have to mark all variables as nullable.
+            // E.g. for
+            // class Foo {
+            //     @Nullable Object a, b;
+            // }
+            // we have to consider both a and b be nullable and write each one
+            // ("Foo\ta" and "Foo\tb")
+            // on a separate line.
+            annot.getLocation().toField().variables.stream()
+                .map(variable -> annot.getLocation().clazz + "\t" + variable + "\n"));
+  }
+
+  /**
+   * Writes the passed annotation to the passed file. It uses the passed mapper to map the
+   * annotation to a stream of strings. And writes each string to a separate line.
+   *
+   * @param annotations Annotations to be written.
+   * @param path Path to the file to be written.
+   * @param mapper Mapper to map the annotation to a stream of strings.
+   */
+  private static void writeAnnotationsToFile(
+      Stream<AddAnnotation> annotations,
+      Path path,
+      Function<AddAnnotation, Stream<String>> mapper) {
+    try (BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(path.toFile()))) {
+      annotations
+          .flatMap(mapper)
+          .forEach(
+              row -> {
+                try {
+                  os.write(row.getBytes(Charset.defaultCharset()), 0, row.length());
+                } catch (IOException e) {
+                  throw new RuntimeException("Error in writing annotation:" + row, e);
+                }
+              });
     } catch (IOException e) {
-      throw new RuntimeException("Error happened for writing at file: " + libraryModelPath, e);
+      throw new RuntimeException("Error happened for writing at file: " + path, e);
     }
   }
 
   /** Removes any existing entry from library models. */
   private void clear() {
     try {
-      new FileOutputStream(libraryModelPath.toFile()).close();
+      Files.deleteIfExists(
+          libraryModelResourcesDirectoryPath.resolve(NULLABLE_FIELD_LIST_FILE_NAME));
+      Files.deleteIfExists(
+          libraryModelResourcesDirectoryPath.resolve(NULLABLE_METHOD_LIST_FILE_NAME));
     } catch (IOException e) {
-      throw new RuntimeException("Could not clear library model loader content", e);
+      throw new RuntimeException(
+          "Error happened for deleting file: " + libraryModelResourcesDirectoryPath, e);
     }
   }
 }

annotator-core/src/test/java/edu/ucr/cs/riple/core/tools/CoreTestHelper.java

@@ -415,15 +415,7 @@ public void makeAnnotatorConfigFile(Path configPath) {
           Utility.getPathToLibraryModel(outDirPath)
               .resolve(
                   Paths.get(
-                      "src",
-                      "main",
-                      "resources",
-                      "edu",
-                      "ucr",
-                      "cs",
-                      "riple",
-                      "librarymodel",
-                      "nullable-methods.tsv"));
+                      "src", "main", "resources", "edu", "ucr", "cs", "riple", "librarymodel"));
     } else {
       builder.buildCommand = projectBuilder.computeTargetBuildCommand(this.outDirPath);
     }

gradle/dependencies.gradle

@@ -42,7 +42,7 @@ def versions = [
         commonsio              : "2.11.0",
         progressbar            : "0.9.2",
         junit                  : "5.7.2",
-        nullaway               : "0.9.8",
+        nullaway               : "0.10.19",
         mockito                : "5.2.0",
 ]
 

library-model-loader/src/main/java/edu/ucr/cs/riple/librarymodel/LibraryModelLoader.java

@@ -22,6 +22,7 @@
 
 package edu.ucr.cs.riple.librarymodel;
 
+import static com.uber.nullaway.LibraryModels.FieldRef.fieldRef;
 import static com.uber.nullaway.LibraryModels.MethodRef.methodRef;
 
 import com.google.auto.service.AutoService;
@@ -38,21 +39,28 @@
 public class LibraryModelLoader implements LibraryModels {
 
   public final String NULLABLE_METHOD_LIST_FILE_NAME = "nullable-methods.tsv";
+  public final String NULLABLE_FIELD_LIST_FILE_NAME = "nullable-fields.tsv";
   public final ImmutableSet<MethodRef> nullableMethods;
+  public final ImmutableSet<FieldRef> nullableFields;
 
   // Assuming this constructor will be called when picked by service loader
   public LibraryModelLoader() {
-    this.nullableMethods = parseTSVFileFromResourcesToMethodRef(NULLABLE_METHOD_LIST_FILE_NAME);
+    this.nullableMethods =
+        parseTSVFileFromResourcesToMemberRef(
+            NULLABLE_METHOD_LIST_FILE_NAME, values -> methodRef(values[0], values[1]));
+    this.nullableFields =
+        parseTSVFileFromResourcesToMemberRef(
+            NULLABLE_FIELD_LIST_FILE_NAME, values -> fieldRef(values[0], values[1]));
   }
 
   /**
-   * Loads a file from resources and parses the content into set of {@link
-   * com.uber.nullaway.LibraryModels.MethodRef}.
+   * Loads a file from resources and creates an instance of type T from each line of the file.
    *
    * @param name File name in resources.
-   * @return ImmutableSet of content in the passed file. Returns empty if the file does not exist.
+   * @return ImmutableSet of contents in the file. Returns empty if the file does not exist.
    */
-  private ImmutableSet<MethodRef> parseTSVFileFromResourcesToMethodRef(String name) {
+  private <T> ImmutableSet<T> parseTSVFileFromResourcesToMemberRef(
+      String name, Factory<T> factory) {
     // Check if resource exists
     if (getClass().getResource(name) == null) {
       return ImmutableSet.of();
@@ -61,13 +69,13 @@ private ImmutableSet<MethodRef> parseTSVFileFromResourcesToMethodRef(String name
       if (is == null) {
         return ImmutableSet.of();
       }
-      ImmutableSet.Builder<MethodRef> contents = ImmutableSet.builder();
+      ImmutableSet.Builder<T> contents = ImmutableSet.builder();
       BufferedReader reader =
           new BufferedReader(new InputStreamReader(is, Charset.defaultCharset()));
       String line = reader.readLine();
       while (line != null) {
         String[] values = line.split("\\t");
-        contents.add(methodRef(values[0], values[1]));
+        contents.add(factory.create(values));
         line = reader.readLine();
       }
       return contents.build();
@@ -120,4 +128,24 @@ public ImmutableSet<MethodRef> nonNullReturns() {
   public ImmutableSetMultimap<MethodRef, Integer> castToNonNullMethods() {
     return ImmutableSetMultimap.of();
   }
+
+  @Override
+  public ImmutableSet<FieldRef> nullableFields() {
+    return nullableFields;
+  }
+
+  /**
+   * Factory interface for creating an instance of type T from a string array.
+   *
+   * @param <T> Type of the instance to create.
+   */
+  interface Factory<T> {
+    /**
+     * Creates an instance of type T from a string array.
+     *
+     * @param values String array to create the instance from.
+     * @return An instance of type T.
+     */
+    T create(String[] values);
+  }
 }