enhancement - Parallelize source set retrieval. (#168)

Co-authored-by: Arthur McGibbon <[email protected]>

model/src/main/java/com/microsoft/java/bs/gradle/model/GradleSourceSet.java

@@ -81,15 +81,20 @@ public interface GradleSourceSet extends Serializable {
   public Set<File> getResourceDirs();
 
   /**
-   * The output directory of this source set.
+   * The output directories of this source set.
    */
-  public File getSourceOutputDir();
+  Set<File> getSourceOutputDirs();
 
   /**
    * The resource output directory of this source set.
    */
   public File getResourceOutputDir();
 
+  /**
+   * Any archive files created from the output of this source set to the output dirs.
+   */
+  Map<File, List<File>> getArchiveOutputFiles();
+
   /**
    * The compile classpath for this source set.
    */

model/src/main/java/com/microsoft/java/bs/gradle/model/LanguageExtension.java

@@ -3,7 +3,9 @@
 
 package com.microsoft.java.bs.gradle.model;
 
+import java.io.File;
 import java.io.Serializable;
+import java.util.Set;
 
 /**
  * Interface representing a language extension.
@@ -13,6 +15,34 @@
  */
 public interface LanguageExtension extends Serializable {
 
+  /**
+   * returns all the source directories for this language.
+   *
+   * @return set of source directories
+   */
+  Set<File> getSourceDirs();
+
+  /**
+   * returns all the generated source directories for this language.
+   *
+   * @return set of generated source directories
+   */
+  Set<File> getGeneratedSourceDirs();
+
+  /**
+   * returns the output directory for this language.
+   *
+   * @return directory containing class files
+   */
+  File getClassesDir();
+
+  /**
+   * returns the name of the Gradle compile task for this language.
+   *
+   * @return name of Gradle compile task
+   */
+  String getCompileTaskName();
+
   /**
    * Checks if the implementing class is a {@link JavaExtension}.
    *
@@ -50,5 +80,4 @@ public interface LanguageExtension extends Serializable {
    *        or null if the cast fails.
    */
   ScalaExtension getAsScalaExtension();
-
 }

model/src/main/java/com/microsoft/java/bs/gradle/model/actions/GetSourceSetsAction.java

@@ -6,23 +6,23 @@
 import com.microsoft.java.bs.gradle.model.BuildTargetDependency;
 import com.microsoft.java.bs.gradle.model.GradleSourceSet;
 import com.microsoft.java.bs.gradle.model.GradleSourceSets;
-import com.microsoft.java.bs.gradle.model.GradleSourceSetsMetadata;
 import com.microsoft.java.bs.gradle.model.impl.DefaultBuildTargetDependency;
 import com.microsoft.java.bs.gradle.model.impl.DefaultGradleSourceSet;
 import com.microsoft.java.bs.gradle.model.impl.DefaultGradleSourceSets;
 import org.gradle.tooling.BuildAction;
 import org.gradle.tooling.BuildController;
+import org.gradle.tooling.model.gradle.BasicGradleProject;
 import org.gradle.tooling.model.gradle.GradleBuild;
 
 import java.io.File;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Set;
-import java.util.Map.Entry;
+import java.util.stream.Collectors;
 
 /**
  * {@link BuildAction} that retrieves {@link DefaultGradleSourceSet} from a Gradle build,
@@ -37,79 +37,144 @@ public class GetSourceSetsAction implements BuildAction<GradleSourceSets> {
    */
   @Override
   public GradleSourceSets execute(BuildController buildController) {
-    Set<String> traversedProjects = new HashSet<>();
-    Map<GradleSourceSet, List<File>> sourceSetToClasspath = new HashMap<>();
-    Map<File, GradleSourceSet> outputsToSourceSet = new HashMap<>();
+    Collection<GradleBuild> builds = fetchIncludedBuilds(buildController);
+    List<GradleSourceSet> sourceSets = fetchModels(buildController, builds);
+    return new DefaultGradleSourceSets(sourceSets);
+  }
 
-    GradleBuild buildModel = buildController.getBuildModel();
-    String rootProjectName = buildModel.getRootProject().getName();
-    fetchModels(buildController,
-        buildModel,
-        traversedProjects,
-        sourceSetToClasspath,
-        outputsToSourceSet,
-        rootProjectName);
-
-    // Add dependencies
-    List<GradleSourceSet> sourceSets = new ArrayList<>();
-    for (Entry<GradleSourceSet, List<File>> entry : sourceSetToClasspath.entrySet()) {
-      Set<BuildTargetDependency> dependencies = new HashSet<>();
-      for (File file : entry.getValue()) {
-        GradleSourceSet otherSourceSet = outputsToSourceSet.get(file);
-        if (otherSourceSet != null && !Objects.equals(entry.getKey(), otherSourceSet)) {
-          dependencies.add(new DefaultBuildTargetDependency(otherSourceSet));
-        }
+  private Collection<GradleBuild> fetchIncludedBuilds(BuildController buildController) {
+    Map<String, GradleBuild> builds = new HashMap<>();
+    GradleBuild build = buildController.getBuildModel();
+    String rootProjectName = build.getRootProject().getName();
+    fetchIncludedBuilds(buildController, build, builds, rootProjectName);
+    return builds.values();
+  }
+
+  private void fetchIncludedBuilds(BuildController buildController, GradleBuild build,
+      Map<String, GradleBuild> builds, String rootProjectName) {
+    if (builds.containsKey(rootProjectName)) {
+      return;
+    }
+    builds.put(rootProjectName, build);
+    // Cannot use GradleVersion.current() in BuildAction as that will return the Tooling API version
+    // Cannot use BuildEnvironment to get GradleVersion as that doesn't work pre-3.0 even though
+    // documentation has it added in version 1.
+    // So just handle exceptions
+    Set<? extends GradleBuild> moreBuilds;
+    try {
+      // added in 4.10
+      moreBuilds = build.getEditableBuilds();
+    } catch (Exception e1) {
+      try {
+        // added in 3.3
+        moreBuilds = build.getIncludedBuilds();
+      } catch (Exception e2) {
+        moreBuilds = null;
       }
+    }
+    if (moreBuilds != null) {
+      for (GradleBuild includedBuild : moreBuilds) {
+        String includedBuildName = includedBuild.getRootProject().getName();
+        fetchIncludedBuilds(buildController, includedBuild, builds, includedBuildName);
+      }
+    }
+  }
 
-      DefaultGradleSourceSet sourceSet = new DefaultGradleSourceSet(entry.getKey());
-      sourceSet.setBuildTargetDependencies(dependencies);
-      sourceSets.add(sourceSet);
+  /**
+   * Fetches source sets from the provided Gradle build model.
+   *
+   * @param buildController The Gradle build controller used to interact with the build.
+   * @param builds The Gradle build models representing the build and included builds.
+   */
+  private List<GradleSourceSet> fetchModels(BuildController buildController,
+      Collection<GradleBuild> builds) {
 
+    List<GetSourceSetAction> projectActions = new ArrayList<>();
+    for (GradleBuild build : builds) {
+      for (BasicGradleProject project : build.getProjects()) {
+        projectActions.add(new GetSourceSetAction(project));
+      }
     }
 
-    return new DefaultGradleSourceSets(sourceSets);
+    // since the model returned from Gradle TAPI is a wrapped object, here we re-construct it
+    // via a copy constructorso we can treat as a DefaultGradleSourceSet and
+    // populate source set dependencies.
+    List<GradleSourceSet> sourceSets = buildController.run(projectActions).stream()
+        .flatMap(ss -> ss.getGradleSourceSets().stream())
+        .map(ss -> new DefaultGradleSourceSet(ss))
+        .collect(Collectors.toList());
+
+    populateInterProjectInfo(sourceSets);
+
+    return sourceSets;
   }
 
   /**
-   * Fetches source sets from the provided Gradle build model and
-   * stores them in a map categorized by project name.
-   *
-   * @param buildController      The Gradle build controller used to interact with the build.
-   * @param build                The Gradle build model representing the current build.
-   * @param traversedProjects    A set of traversed project names to avoid cyclic dependencies.
-   * @param sourceSetToClasspath A map that associates GradleSourceSet objects with their
-   *                             corresponding classpath files.
-   * @param outputsToSourceSet   A map that associates output files with the GradleSourceSet
-   *                             they belong to.
-   * @param buildName            The name of the root project in the build.
+   * {@link BuildAction} that retrieves {@link GradleSourceSets} for a single project.
+   * This allows project models to be retrieved in parallel.
    */
-  private void fetchModels(
-      BuildController buildController,
-      GradleBuild build,
-      Set<String> traversedProjects,
-      Map<GradleSourceSet, List<File>> sourceSetToClasspath,
-      Map<File, GradleSourceSet> outputsToSourceSet,
-      String buildName
-  ) {
-    if (traversedProjects.contains(buildName)) {
-      return;
+  static class GetSourceSetAction implements BuildAction<GradleSourceSets> {
+    private final BasicGradleProject project;
+
+    public GetSourceSetAction(BasicGradleProject project) {
+      this.project = project;
     }
-    GradleSourceSetsMetadata sourceSets = buildController
-        .findModel(build.getRootProject(), GradleSourceSetsMetadata.class);
-
-    traversedProjects.add(buildName);
-    sourceSetToClasspath.putAll(sourceSets.getGradleSourceSetsToClasspath());
-    outputsToSourceSet.putAll(sourceSets.getOutputsToSourceSet());
-
-    for (GradleBuild includedBuild : build.getIncludedBuilds()) {
-      String includedBuildName = includedBuild.getRootProject().getName();
-      fetchModels(buildController,
-          includedBuild,
-          traversedProjects,
-          sourceSetToClasspath,
-          outputsToSourceSet,
-          includedBuildName);
+
+    @Override
+    public GradleSourceSets execute(BuildController controller) {
+      return controller.getModel(project, GradleSourceSets.class);
     }
   }
 
-}
+  // Inter-sourceset dependencies must be built up after retrieval of all sourcesets
+  // because they are not available before when using included builds.
+  // Classpaths that reference other projects using jars are to be replaced with
+  // source paths.
+  private void populateInterProjectInfo(List<GradleSourceSet> sourceSets) {
+    // map all output dirs to their source sets
+    Map<File, List<File>> archivesToSourceOutput = new HashMap<>();
+    Map<File, GradleSourceSet> outputsToSourceSet = new HashMap<>();
+    for (GradleSourceSet sourceSet : sourceSets) {
+      if (sourceSet.getSourceOutputDirs() != null) {
+        for (File file : sourceSet.getSourceOutputDirs()) {
+          outputsToSourceSet.put(file, sourceSet);
+        }
+      }
+      if (sourceSet.getResourceOutputDir() != null) {
+        outputsToSourceSet.put(sourceSet.getResourceOutputDir(), sourceSet);
+      }
+      if (sourceSet.getArchiveOutputFiles() != null) {
+        for (Map.Entry<File, List<File>> archive : sourceSet.getArchiveOutputFiles().entrySet()) {
+          outputsToSourceSet.put(archive.getKey(), sourceSet);
+          archivesToSourceOutput.computeIfAbsent(archive.getKey(), f -> new ArrayList<>())
+              .addAll(archive.getValue());
+        }
+      }
+    }
+
+    // match any classpath entries to other project's output dirs/jars to create dependencies.
+    // replace classpath entries that reference jars with classes dirs.
+    for (GradleSourceSet sourceSet : sourceSets) {
+      Set<BuildTargetDependency> dependencies = new HashSet<>();
+      List<File> classpath = new ArrayList<>();
+      for (File file : sourceSet.getCompileClasspath()) {
+        // add project dependency
+        GradleSourceSet otherSourceSet = outputsToSourceSet.get(file);
+        if (otherSourceSet != null) {
+          dependencies.add(new DefaultBuildTargetDependency(otherSourceSet));
+        }
+        // replace jar on classpath with source output on classpath
+        List<File> sourceOutputDir = archivesToSourceOutput.get(file);
+        if (sourceOutputDir == null) {
+          classpath.add(file);
+        } else {
+          classpath.addAll(sourceOutputDir);
+        }
+      }
+      if (sourceSet instanceof DefaultGradleSourceSet) {
+        ((DefaultGradleSourceSet) sourceSet).setBuildTargetDependencies(dependencies);
+        ((DefaultGradleSourceSet) sourceSet).setCompileClasspath(classpath);
+      }
+    }
+  }
+}