Add On Import Translation of Python Files in Include Path

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationManager.kt

@@ -39,11 +39,13 @@
 import java.io.PrintWriter
 import java.lang.reflect.InvocationTargetException
 import java.nio.file.Files
+import java.nio.file.Path
 import java.util.*
 import java.util.concurrent.CompletableFuture
 import java.util.concurrent.CompletionException
 import java.util.concurrent.ExecutionException
 import java.util.concurrent.atomic.AtomicBoolean
+import kotlin.io.path.name
 import kotlin.reflect.full.findAnnotation
 import org.slf4j.LoggerFactory
 
@@ -140,15 +142,24 @@
         result: TranslationResult,
     ): Set<LanguageFrontend<*, *>> {
         val usedFrontends = mutableSetOf<LanguageFrontend<*, *>>()
+
+        val externalSources: MutableList<File> = mutableListOf()
+        val importedSources: MutableSet<File> = mutableSetOf()
+
+        ctx.config.includePaths.forEach { externalSources.addAll(extractConfiguredSources(it)) }
+
+        var useParallelFrontends = ctx.config.useParallelFrontends
+
         for (sc in ctx.config.softwareComponents.keys) {
             val component = Component()
             component.ctx = ctx
             component.name = Name(sc)
             result.addComponent(component)
 
+            // Every Component needs to reprocess the sources
             var sourceLocations: List<File> = ctx.config.softwareComponents[sc] ?: listOf()
 
-            var useParallelFrontends = ctx.config.useParallelFrontends
+            // Todo Here we need to dispatch to the correct frontend to do preproccessing
 
             val list =
                 sourceLocations.flatMap { file ->
@@ -172,6 +183,7 @@
                         files
                     } else {
                         val frontendClass = file.language?.frontend
+
                         val supportsParallelParsing =
                             file.language
                                 ?.frontend
@@ -243,6 +255,29 @@
                 sourceLocations = list
             }
 
+            val processedImports: MutableList<String> = mutableListOf()
+
+            sourceLocations
+                .filter { it.language != null }
+                .forEach {
+                    val frontend = it.language?.newFrontend(ctx)
+                    importedSources.addAll(
+                        frontend?.gatherExternalSources(
+                            ctx.config.includePaths,
+                            it,
+                            externalSources,
+                            processedImports,
+                        ) ?: listOf()
+                    )
+                }
+            log.info(
+                "Extending " +
+                    sourceLocations.size +
+                    " source files by " +
+                    importedSources.size +
+                    " interface files."
+            )
+
             usedFrontends.addAll(
                 if (useParallelFrontends) {
                     parseParallel(component, result, ctx, sourceLocations)
@@ -252,9 +287,43 @@
             )
         }
 
+        // Distribute all files by there root path prefix, parse them in individual component
+        // named
+        // like their rootPath local name
+        ctx.config.includePaths.forEach { includePath ->
+            val filesInPath =
+                importedSources.filter {
+                    it.path.removePrefix(includePath.toString()) != it.path.toString()
+                }
+            if (filesInPath.isNotEmpty()) {
+                val component = Component()
+                component.ctx = ctx
+                component.name = Name(includePath.name)
+                result.addComponent(component)
+
+                ctx.config.topLevels.put(includePath.toString(), includePath.toFile())
+
+                usedFrontends.addAll(
+                    if (useParallelFrontends) {
+                        parseParallel(component, result, ctx, filesInPath)
+                    } else {
+                        parseSequentially(component, result, ctx, filesInPath)
+                    }
+                )
+            }
+        }
+
         return usedFrontends
     }
 
+    private fun extractConfiguredSources(path: Path): MutableList<File> {
+        val rootFile = path.toFile()
+        return if (rootFile.exists())
+            (if (rootFile.isDirectory) rootFile.walkTopDown().toMutableList()
+            else mutableListOf(rootFile))
+        else mutableListOf()
+    }
+
     private fun parseParallel(
         component: Component,
         result: TranslationResult,

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageFrontend.kt

@@ -33,6 +33,7 @@
 import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation
 import de.fraunhofer.aisec.cpg.sarif.Region
 import java.io.File
+import java.nio.file.Path
 import java.util.*
 import org.slf4j.LoggerFactory
 
@@ -110,6 +111,13 @@
      */
     abstract override fun codeOf(astNode: AstNode): String?
 
+    abstract fun gatherExternalSources(
+        rootPaths: List<Path>,
+        source: File,
+        externalSources: MutableList<File>,
+        processedImports: MutableList<String> = mutableListOf<String>(),
+    ): List<File>
+
     /**
      * Returns the [Region] of the code with line and column, index starting at 1, generic for java
      * or c++ ast nodes.

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Component.kt

@@ -34,6 +34,7 @@ import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf
 import de.fraunhofer.aisec.cpg.graph.edges.unwrapping
 import de.fraunhofer.aisec.cpg.passes.ImportDependencies
 import de.fraunhofer.aisec.cpg.passes.ImportResolver
+import de.fraunhofer.aisec.cpg.persistence.DoNotPersist
 import java.io.File
 import org.neo4j.ogm.annotation.Relationship
 import org.neo4j.ogm.annotation.Transient
@@ -64,6 +65,7 @@ open class Component : Node() {
      * Returns the top-level directory of this component according to
      * [TranslationConfiguration.topLevels]
      */
+    @DoNotPersist
     val topLevel: File?
         get() {
             return ctx?.config?.topLevels?.get(this.name.localName)

cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/frontends/TestLanguage.kt

@@ -34,6 +34,7 @@ import de.fraunhofer.aisec.cpg.graph.types.*
 import de.fraunhofer.aisec.cpg.graph.unknownType
 import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation
 import java.io.File
+import java.nio.file.Path
 import java.util.function.Supplier
 import kotlin.reflect.KClass
 
@@ -95,6 +96,15 @@ open class TestLanguageFrontend(
         TODO("Not yet implemented")
     }
 
+    override fun gatherExternalSources(
+        rootPaths: List<Path>,
+        source: File,
+        externalSources: MutableList<File>,
+        processedImports: MutableList<String>,
+    ): List<File> {
+        return listOf()
+    }
+
     override fun locationOf(astNode: Any): PhysicalLocation? {
         TODO("Not yet implemented")
     }

cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontend.kt

@@ -804,6 +804,15 @@ open class CXXLanguageFrontend(language: Language<CXXLanguageFrontend>, ctx: Tra
         return type
     }
 
+    override fun gatherExternalSources(
+        rootPaths: List<Path>,
+        source: File,
+        externalSources: MutableList<File>,
+        processedImports: MutableList<String>,
+    ): List<File> {
+        return listOf()
+    }
+
     companion object {
         private val LOGGER = LoggerFactory.getLogger(CXXLanguageFrontend::class.java)
 

cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt

@@ -51,6 +51,7 @@ import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation
 import de.fraunhofer.aisec.cpg.sarif.Region
 import java.io.File
 import java.net.URI
+import java.nio.file.Path
 
 /**
  * A language frontend for the [GoLanguage]. It makes use the internal
@@ -419,6 +420,15 @@ class GoLanguageFrontend(language: Language<GoLanguageFrontend>, ctx: Translatio
         }
     }
 
+    override fun gatherExternalSources(
+        rootPaths: List<Path>,
+        source: File,
+        externalSources: MutableList<File>,
+        processedImports: MutableList<String>,
+    ): List<File> {
+        return listOf()
+    }
+
     companion object {
         /**
          * All possible goos values. See

cpg-language-ini/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/ini/IniFileFrontend.kt

@@ -42,6 +42,7 @@ import de.fraunhofer.aisec.cpg.sarif.Region
 import java.io.File
 import java.io.FileInputStream
 import java.net.URI
+import java.nio.file.Path
 import org.ini4j.Ini
 import org.ini4j.Profile.Section
 
@@ -205,4 +206,13 @@ class IniFileFrontend(language: Language<IniFileFrontend>, ctx: TranslationConte
     override fun setComment(node: Node, astNode: Any) {
         return // not used as this function does not implement [Handler]
     }
+
+    override fun gatherExternalSources(
+        rootPaths: List<Path>,
+        source: File,
+        externalSources: MutableList<File>,
+        processedImports: MutableList<String>,
+    ): List<File> {
+        return listOf()
+    }
 }

cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguageFrontend.kt

@@ -71,6 +71,7 @@ import de.fraunhofer.aisec.cpg.sarif.Region
 import java.io.File
 import java.io.FileNotFoundException
 import java.io.IOException
+import java.nio.file.Path
 import java.util.function.Consumer
 import kotlin.jvm.optionals.getOrNull
 
@@ -514,6 +515,15 @@ open class JavaLanguageFrontend(language: Language<JavaLanguageFrontend>, ctx: T
         }
     }
 
+    override fun gatherExternalSources(
+        rootPaths: List<Path>,
+        source: File,
+        externalSources: MutableList<File>,
+        processedImports: MutableList<String>,
+    ): List<File> {
+        return listOf()
+    }
+
     companion object {
         const val THIS = "this"
         const val ANNOTATION_MEMBER_VALUE = "value"

cpg-language-jvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/jvm/JVMLanguageFrontend.kt

@@ -35,6 +35,7 @@ import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration
 import de.fraunhofer.aisec.cpg.graph.types.Type
 import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation
 import java.io.File
+import java.nio.file.Path
 import sootup.core.model.Body
 import sootup.core.model.SootMethod
 import sootup.core.model.SourceType
@@ -190,4 +191,13 @@ class JVMLanguageFrontend(
             }
         }
     }
+
+    override fun gatherExternalSources(
+        rootPaths: List<Path>,
+        source: File,
+        externalSources: MutableList<File>,
+        processedImports: MutableList<String>,
+    ): List<File> {
+        return listOf()
+    }
 }

cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/LLVMIRLanguageFrontend.kt

@@ -43,6 +43,7 @@ import de.fraunhofer.aisec.cpg.passes.configuration.RegisterExtraPass
 import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation
 import java.io.File
 import java.nio.ByteBuffer
+import java.nio.file.Path
 import org.bytedeco.javacpp.BytePointer
 import org.bytedeco.javacpp.Pointer
 import org.bytedeco.llvm.LLVM.*
@@ -264,6 +265,15 @@ class LLVMIRLanguageFrontend(language: Language<LLVMIRLanguageFrontend>, ctx: Tr
             ""
         }
     }
+
+    override fun gatherExternalSources(
+        rootPaths: List<Path>,
+        source: File,
+        externalSources: MutableList<File>,
+        processedImports: MutableList<String>,
+    ): List<File> {
+        return listOf()
+    }
 }
 
 /**