Skip to content

Commit

Permalink
[x2cpg] Fix a few Gradle dependency fetching issues (#5333)
Browse files Browse the repository at this point in the history 
* Make Gradle dependency fetcher init script more robust

* Fetch class jars instead of source jars in init script
  • Loading branch information
@johannescoetzee
johannescoetzee authored Mar 3, 2025
1 parent 13c4037 commit 8123925
Show file tree
Hide file tree
Showing 6 changed files with 86 additions and 47 deletions.
Empty file added joern-cli/frontends/javasrc2cpg/src/test/resources/config_tests/properties.gradle
View file
Empty file.
Empty file added joern-cli/frontends/javasrc2cpg/src/test/resources/config_tests/settings.gradle
View file
Empty file.
2 changes: 2 additions & 0 deletions .../javasrc2cpg/src/test/scala/io/joern/javasrc2cpg/passes/ConfigFileCreationPassTests.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ class ConfigFileCreationPassTests extends JavaSrcCode2CpgFixture {
Paths.get(absoluteConfigDir, "struts.xml").toString,
Paths.get(absoluteConfigDir, "web.xml").toString,
Paths.get(absoluteConfigDir, "build.gradle").toString,
Paths.get(absoluteConfigDir, "settings.gradle").toString,
Paths.get(absoluteConfigDir, "properties.gradle").toString,
Paths.get(absoluteConfigDir, "build.gradle.kts").toString,
Paths.get(absoluteConfigDir, "basic.yaml").toString,
Paths.get(absoluteConfigDir, "basic.yml").toString,
Expand Down
4 changes: 2 additions & 2 deletions ...ontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XConfigFileCreationPass.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,8 @@ class JavaConfigFileCreationPass(cpg: Cpg, rootDir: Option[String] = None)
// MYBATIS
mybatisFilter,
// BUILD SYSTEM
pathEndFilter("build.gradle"),
pathEndFilter("build.gradle.kts"),
extensionFilter(".gradle"),
pathEndFilter(".gradle.kts"),
// ANDROID
pathEndFilter("AndroidManifest.xml"),
// SPRING
Expand Down
2 changes: 1 addition & 1 deletion ...i/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/dependency/DependencyResolver.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ object DependencyResolver {

GradleDependencies.get(projectDir, maybeProjectNameOverride, maybeConfigurationOverride) match {
case dependenciesMap if dependenciesMap.values.exists(_.nonEmpty) =>
Option(dependenciesMap.values.flatten.toSet.toSeq)
Option(dependenciesMap.values.flatten.toList.distinctBy(path => path.split(java.io.File.separatorChar).last))

case _ =>
logger.warn(s"Could not download Gradle dependencies for project at path `$projectDir`")
Expand Down
125 changes: 81 additions & 44 deletions ...i/frontends/x2cpg/src/main/scala/io/joern/x2cpg/utils/dependency/GradleDependencies.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,13 @@ import scala.collection.mutable
import scala.jdk.CollectionConverters.*
import scala.util.{Failure, Random, Success, Try, Using}

case class ProjectNameInfo(projectName: String, isSubproject: Boolean) {
case class ProjectNameInfo(projectName: String, parentName: Option[String]) {
override def toString: String = {
if (isSubproject)
s":$projectName"
else
projectName
parentName.map(parentName => s"$parentName:$projectName").getOrElse(projectName)
}

def makeGradleTaskName(taskName: String): String = {
if (isSubproject)
s"$projectName:$taskName"
else
taskName
parentName.map(parentName => s"$parentName:$projectName:$taskName").getOrElse(taskName)
}
}

Expand Down Expand Up @@ -81,49 +75,85 @@ object GradleDependencies {
val androidTaskDefinition = Option.when(projectInfo.hasAndroidSubproject)(s"""
|def androidDepsCopyTaskName = taskName + "_androidDeps"
| $taskCreationFunction(androidDepsCopyTaskName, Copy) {
| def paths = project.configurations.find { it.name.equals("androidApis") }
| if (paths == null) paths = []
| duplicatesStrategy = 'include'
| into destinationDir
| from project.configurations.find { it.name.equals("androidApis") }
| from paths
| }
|""".stripMargin)

val dependsOnAndroidTask = Option.when(projectInfo.hasAndroidSubproject)("dependsOn androidDepsCopyTaskName")

s"""
|allprojects { project ->
| def taskName = "$taskName"
| def destinationDir = "${destination.replaceAll("\\\\", "/")}"
| def gradleProjectConfigurations = [$projectConfigurationString]
s"""import java.nio.file.Path
|import java.nio.file.Files
|import java.nio.file.StandardCopyOption
|
| if (gradleProjectConfigurations.containsKey(project.name)) {
| def gradleConfigurationNames = gradleProjectConfigurations.get(project.name)
|abstract class FetchDependencies extends DefaultTask {
|
| def compileDepsCopyTaskName = taskName + "_compileDeps"
| $taskCreationFunction(compileDepsCopyTaskName, Copy) {
| FetchDependencies() {
| outputs.upToDateWhen { false }
| }
|
| def selectedConfigs = project.configurations.findAll {
| configuration -> gradleConfigurationNames.contains(configuration.getName())
| }
| @Input
| abstract ListProperty<String> getSelectedConfigNames()
|
| def componentIds = []
| if (!selectedConfigs.isEmpty()) {
| for (selectedConfig in selectedConfigs) {
| componentIds = selectedConfig.incoming.resolutionResult.allDependencies.findAll {
| dep -> dep instanceof org.gradle.api.internal.artifacts.result.DefaultResolvedDependencyResult
| } .collect { it.selected.id }
| @OutputDirectory
| abstract Property<String> getDestinationDirString()
|
| @TaskAction
| void fetch() {
| def projectString = project.toString()
| def projectFullName = ""
| if (projectString.startsWith("root project")) {
| projectFullName = projectString.substring("root project '".length(), projectString.length() - 1)
| } else {
| projectFullName = projectString.substring("project ':".length(), projectString.length() - 1)
| }
|
| def destinationDir = Path.of(destinationDirString.get(), projectFullName.replace(':', '/'))
| Files.createDirectories(destinationDir)
| def selectedConfigs = project.configurations.findAll {
| configuration -> selectedConfigNames.get().contains(configuration.getName())
| }
|
| for (selectedConfig in selectedConfigs) {
| // See https://docs.gradle.org/current/userguide/artifact_resolution.html#artifact-resolution for more information
| for (artifactFiles in selectedConfig.incoming.artifacts.resolvedArtifacts.map{ it.collect { artifact -> artifact.file } }) {
| for (file in artifactFiles.get()) {
| try {
| Files.createSymbolicLink(
| destinationDir.resolve(file.name),
| Path.of(file.getPath())
| )
| } catch (Exception e) {
| Files.copy(
| Path.of(file.getPath()),
| destinationDir.resolve(file.name),
| StandardCopyOption.REPLACE_EXISTING
| )
| }
| }
| }
| }
| }
|}
|
| def result = dependencies.createArtifactResolutionQuery()
| .forComponents(componentIds)
| .withArtifacts(JvmLibrary, SourcesArtifact)
| .execute()
| duplicatesStrategy = 'include'
| into destinationDir
| from result.resolvedComponents.collect { it.getArtifacts(SourcesArtifact).collect { it.file } }
|allprojects { project ->
| def taskName = "$taskName"
| def compileDepsCopyTaskName = taskName + "_compileDeps"
| def destinationDir = "${destination.replaceAll("\\\\", "/")}"
| def gradleProjectConfigurations = [$projectConfigurationString]
|
| if (gradleProjectConfigurations.containsKey(project.name)) {
| $taskCreationFunction(compileDepsCopyTaskName, FetchDependencies) {
| selectedConfigNames = gradleProjectConfigurations.get(project.name)
| destinationDirString = destinationDir
| }
|
| ${androidTaskDefinition.getOrElse("")}
| $taskCreationFunction(taskName, Copy) {
|
| $taskCreationFunction(taskName) {
| ${dependsOnAndroidTask.getOrElse("")}
| dependsOn compileDepsCopyTaskName
| }
Expand Down Expand Up @@ -167,6 +197,13 @@ object GradleDependencies {
results.filterNot(_.toLowerCase.contains("test")).toList
}

private def getSubprojectNames(project: GradleProject, parentName: String): List[ProjectNameInfo] = {
project.getChildren.asScala.flatMap { childProject =>
val childNameInfo = ProjectNameInfo(childProject.getName, Option(parentName))
childNameInfo :: getSubprojectNames(childProject, childNameInfo.toString)
}.toList
}

private def getGradleProjectInfo(
projectDir: Path,
projectNameOverride: Option[String],
Expand All @@ -179,9 +216,7 @@ object GradleDependencies {
val buildEnv = connection.getModel[BuildEnvironment](classOf[BuildEnvironment])
val project = connection.getModel[GradleProject](classOf[GradleProject])

val availableProjectNames = ProjectNameInfo(project.getName, false) :: project.getChildren.asScala
.map(child => ProjectNameInfo(child.getName, true))
.toList
val availableProjectNames = ProjectNameInfo(project.getName, None) :: getSubprojectNames(project, "")

val availableProjectNamesString = availableProjectNames.mkString(" ")

Expand Down Expand Up @@ -362,6 +397,7 @@ object GradleDependencies {
.list(destinationDir)
.collect(Collectors.toList[Path])
.asScala
.filterNot(File(_).isDirectory)
.map(_.toAbsolutePath.toString)
logger.info(s"Task $taskName resolved `${result.size}` dependency files.")
Some(result)
Expand All @@ -375,9 +411,6 @@ object GradleDependencies {
}
if (stderrStream.toString.contains("Could not compile initialization script")) {
val scriptContents = File(initScriptPath).contentAsString
logger.debug(
s"########## INITIALIZATION_SCRIPT ##########\n$scriptContents\n###########################################"
)
}
logger.debug(s"Gradle task execution stdout: \n$stdoutStream")
logger.debug(s"Gradle task execution stderr: \n$stderrStream")
Expand Down Expand Up @@ -436,8 +469,13 @@ object GradleDependencies {
Using.resource(connection) { c =>
projectInfo.subprojects.keys.flatMap { projectNameInfo =>
val taskName = projectNameInfo.makeGradleTaskName(initScript.taskName)
val destinationSubdir = projectNameInfo.parentName match {
case None => projectNameInfo.projectName
case _ => projectNameInfo.toString.stripPrefix(":").replace(':', '/')
}
val destinationDir = initScript.destinationDir.resolve(destinationSubdir)

runGradleTask(c, taskName, initScript.destinationDir, initScriptFile.pathAsString) map { deps =>
runGradleTask(c, taskName, destinationDir, initScriptFile.pathAsString) map { deps =>
val depsOutput = deps.map { d =>
if (!d.endsWith(aarFileExtension)) d
else
Expand All @@ -446,7 +484,6 @@ object GradleDependencies {
case None => d
}
}

projectNameInfo.projectName -> depsOutput
}
}.toMap
Expand Down

0 comments on commit 8123925

Please sign in to comment.