Skip to content

Commit 77d5da2

Browse files
committed
Playing around with an EOG passed concept pass based on tasks
1 parent 7d6bbdd commit 77d5da2

File tree

17 files changed

+340
-174
lines changed

17 files changed

+340
-174
lines changed

cpg-concepts/src/integrationTest/kotlin/de/fraunhofer/aisec/cpg/concepts/ConfigurationPassTest.kt

+12-6
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,11 @@ import de.fraunhofer.aisec.cpg.graph.concepts.config.ReadConfigurationOption
3939
import de.fraunhofer.aisec.cpg.graph.concepts.config.RegisterConfigurationGroup
4040
import de.fraunhofer.aisec.cpg.graph.concepts.config.RegisterConfigurationOption
4141
import de.fraunhofer.aisec.cpg.graph.statements.expressions.SubscriptExpression
42-
import de.fraunhofer.aisec.cpg.passes.concepts.config.ProvideConfigPass
43-
import de.fraunhofer.aisec.cpg.passes.concepts.config.ini.IniFileConfigurationSourcePass
44-
import de.fraunhofer.aisec.cpg.passes.concepts.config.python.PythonStdLibConfigurationPass
42+
import de.fraunhofer.aisec.cpg.passes.PassConfiguration
43+
import de.fraunhofer.aisec.cpg.passes.concepts.ConceptPass
44+
import de.fraunhofer.aisec.cpg.passes.concepts.config.ProvideConfigTask
45+
import de.fraunhofer.aisec.cpg.passes.concepts.config.ini.IniFileConfigurationSourceTask
46+
import de.fraunhofer.aisec.cpg.passes.concepts.config.python.PythonStdLibConfigurationTask
4547
import de.fraunhofer.aisec.cpg.test.analyze
4648
import java.io.File
4749
import kotlin.test.Test
@@ -57,9 +59,13 @@ class ConfigurationPassTest {
5759
analyze(listOf(), topLevel.toPath(), true) {
5860
it.registerLanguage<PythonLanguage>()
5961
it.registerLanguage<IniFileLanguage>()
60-
it.registerPass<IniFileConfigurationSourcePass>()
61-
it.registerPass<PythonStdLibConfigurationPass>()
62-
it.registerPass<ProvideConfigPass>()
62+
it.registerPass<ConceptPass>()
63+
it.configurePass<ConceptPass>(
64+
PassConfiguration()
65+
.registerTask<IniFileConfigurationSourceTask>()
66+
.registerTask<PythonStdLibConfigurationTask>()
67+
.registerTask<ProvideConfigTask>()
68+
)
6369
it.softwareComponents(
6470
mutableMapOf(
6571
"conf" to listOf(topLevel.resolve("conf")),

cpg-concepts/src/integrationTest/kotlin/de/fraunhofer/aisec/cpg/concepts/DynamicLoadingTest.kt

+20-3
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,11 @@ import de.fraunhofer.aisec.cpg.graph.concepts.memory.LoadLibrary
3333
import de.fraunhofer.aisec.cpg.graph.concepts.memory.LoadSymbol
3434
import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration
3535
import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression
36-
import de.fraunhofer.aisec.cpg.passes.concepts.memory.cxx.CXXDynamicLoadingPass
36+
import de.fraunhofer.aisec.cpg.passes.ImportResolver
37+
import de.fraunhofer.aisec.cpg.passes.PassConfiguration
38+
import de.fraunhofer.aisec.cpg.passes.concepts.ConceptPass
39+
import de.fraunhofer.aisec.cpg.passes.concepts.memory.cxx.CXXDynamicLoadingImportTask
40+
import de.fraunhofer.aisec.cpg.passes.concepts.memory.cxx.CXXDynamicLoadingTask
3741
import de.fraunhofer.aisec.cpg.test.analyze
3842
import de.fraunhofer.aisec.cpg.test.assertInvokes
3943
import java.io.File
@@ -50,7 +54,13 @@ class DynamicLoadingTest {
5054
val result =
5155
analyze(listOf(), topLevel.toPath(), true) {
5256
it.registerLanguage<CLanguage>()
53-
it.registerPass<CXXDynamicLoadingPass>()
57+
it.registerPass<ConceptPass>()
58+
it.configurePass<ConceptPass>(
59+
PassConfiguration().registerTask<CXXDynamicLoadingTask>()
60+
)
61+
it.configurePass<ImportResolver>(
62+
PassConfiguration().registerTask<CXXDynamicLoadingImportTask>()
63+
)
5464
it.softwareComponents(
5565
mutableMapOf(
5666
"main" to listOf(topLevel.resolve("main")),
@@ -120,7 +130,13 @@ class DynamicLoadingTest {
120130
val result =
121131
analyze(listOf(), topLevel.toPath(), true) {
122132
it.registerLanguage<CLanguage>()
123-
it.registerPass<CXXDynamicLoadingPass>()
133+
it.registerPass<ConceptPass>()
134+
it.configurePass<ConceptPass>(
135+
PassConfiguration().registerTask<CXXDynamicLoadingTask>()
136+
)
137+
it.configurePass<ImportResolver>(
138+
PassConfiguration().registerTask<CXXDynamicLoadingImportTask>()
139+
)
124140
it.softwareComponents(
125141
mutableMapOf(
126142
"winmain" to listOf(topLevel.resolve("winmain")),
@@ -146,6 +162,7 @@ class DynamicLoadingTest {
146162
loadLibrary.what,
147163
"\"what\" of the LoadLibrary should be the winexample component",
148164
)
165+
assertEquals(1, loadLibrary.entryPoints.size)
149166
assertEquals(
150167
dllMain,
151168
loadLibrary.entryPoints.singleOrNull()?.underlyingNode,

cpg-concepts/src/integrationTest/kotlin/de/fraunhofer/aisec/cpg/concepts/EntryPointTest.kt

+5-2
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,9 @@ import de.fraunhofer.aisec.cpg.graph.concepts.arch.POSIX
3131
import de.fraunhofer.aisec.cpg.graph.concepts.arch.Win32
3232
import de.fraunhofer.aisec.cpg.graph.concepts.flows.Main
3333
import de.fraunhofer.aisec.cpg.graph.get
34-
import de.fraunhofer.aisec.cpg.passes.concepts.flows.cxx.CXXEntryPointsPass
34+
import de.fraunhofer.aisec.cpg.passes.PassConfiguration
35+
import de.fraunhofer.aisec.cpg.passes.concepts.ConceptPass
36+
import de.fraunhofer.aisec.cpg.passes.concepts.flows.cxx.CXXEntryPointTask
3537
import de.fraunhofer.aisec.cpg.test.analyze
3638
import java.io.File
3739
import kotlin.test.assertIs
@@ -45,7 +47,8 @@ class EntryPointTest {
4547
val result =
4648
analyze(listOf(), topLevel.toPath(), true) {
4749
it.registerLanguage<CLanguage>()
48-
it.registerPass<CXXEntryPointsPass>()
50+
it.registerPass<ConceptPass>()
51+
it.configurePass<ConceptPass>(PassConfiguration().registerTask<CXXEntryPointTask>())
4952
it.softwareComponents(
5053
mutableMapOf(
5154
"main" to listOf(topLevel.resolve("main")),

cpg-concepts/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/concepts/ConceptPass.kt

+53-30
Original file line numberDiff line numberDiff line change
@@ -26,63 +26,86 @@
2626
package de.fraunhofer.aisec.cpg.passes.concepts
2727

2828
import de.fraunhofer.aisec.cpg.TranslationContext
29+
import de.fraunhofer.aisec.cpg.graph.Component
2930
import de.fraunhofer.aisec.cpg.graph.Node
3031
import de.fraunhofer.aisec.cpg.graph.allEOGStarters
31-
import de.fraunhofer.aisec.cpg.graph.component
3232
import de.fraunhofer.aisec.cpg.graph.conceptNodes
3333
import de.fraunhofer.aisec.cpg.graph.concepts.Concept
3434
import de.fraunhofer.aisec.cpg.graph.concepts.Operation
3535
import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration
3636
import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker
37-
import de.fraunhofer.aisec.cpg.passes.TranslationUnitPass
37+
import de.fraunhofer.aisec.cpg.passes.ComponentPass
38+
import de.fraunhofer.aisec.cpg.passes.ControlFlowSensitiveDFGPass
39+
import de.fraunhofer.aisec.cpg.passes.DynamicInvokeResolver
40+
import de.fraunhofer.aisec.cpg.passes.SymbolResolver
41+
import de.fraunhofer.aisec.cpg.passes.Task
42+
import de.fraunhofer.aisec.cpg.passes.configuration.DependsOn
43+
import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteBefore
3844
import de.fraunhofer.aisec.cpg.processing.strategy.Strategy
3945

46+
typealias ConceptTask = Task<Component, ConceptPass>
47+
4048
/**
41-
* An abstract pass that is used to identify and create [Concept] and [Operation] nodes in the
42-
* graph.
49+
* A pass that is used to identify and create [Concept] and [Operation] nodes in the graph. The idea
50+
* is that developers that add concepts do not need to implement a full pass, but can just implement
51+
* a small [Task] that handles the nodes relevant for the concept.
4352
*/
44-
abstract class ConceptPass(ctx: TranslationContext) : TranslationUnitPass(ctx) {
53+
@DependsOn(SymbolResolver::class)
54+
@DependsOn(ControlFlowSensitiveDFGPass::class)
55+
@ExecuteBefore(DynamicInvokeResolver::class)
56+
class ConceptPass(ctx: TranslationContext) : ComponentPass(ctx) {
4557

4658
lateinit var walker: SubgraphWalker.ScopedWalker
4759

48-
override fun accept(tu: TranslationUnitDeclaration) {
49-
ctx.currentComponent = tu.component
60+
override fun accept(c: Component) {
61+
ctx.currentComponent = c
5062
walker = SubgraphWalker.ScopedWalker(ctx.scopeManager)
5163
walker.strategy = Strategy::EOG_FORWARD
52-
walker.registerHandler { node -> handleNode(node, tu) }
64+
walker.registerHandler { node -> handleNode(node) }
65+
66+
// Process nodes in our translation units in the order depending on their import
67+
// dependencies
68+
for (tu in (Strategy::TRANSLATION_UNITS_LEAST_IMPORTS)(c)) {
69+
log.debug("Processing concepts of translation unit {}", tu.name)
5370

54-
// Gather all resolution EOG starters; and make sure they really do not have a
55-
// predecessor, otherwise we might analyze a node multiple times
56-
val nodes = tu.allEOGStarters.filter { it.prevEOGEdges.isEmpty() }
71+
// Gather all resolution EOG starters; and make sure they really do not have a
72+
// predecessor, otherwise we might analyze a node multiple times
73+
val nodes = tu.allEOGStarters.filter { it.prevEOGEdges.isEmpty() }
5774

58-
for (node in nodes) {
59-
walker.iterate(node)
75+
for (node in nodes) {
76+
walker.iterate(node)
77+
}
6078
}
6179
}
6280

6381
/**
64-
* This function is called for each node in the graph. It needs to be overridden by subclasses
65-
* to handle the specific node.
66-
*/
67-
abstract fun handleNode(node: Node, tu: TranslationUnitDeclaration)
68-
69-
/**
70-
* Gets concept of type [T] for this [TranslationUnitDeclaration] or creates a new one if it
71-
* does not exist.
82+
* This function is called for each node in the graph. It will invoke all plugins that are
83+
* registered in the pass configuration.
7284
*/
73-
internal inline fun <reified T : Concept> TranslationUnitDeclaration.getConceptOrCreate(
74-
noinline init: ((T) -> Unit)? = null
75-
): T {
76-
var concept = this.conceptNodes.filterIsInstance<T>().singleOrNull()
77-
if (concept == null) {
78-
concept = T::class.constructors.first().call(this)
79-
init?.invoke(concept)
85+
fun handleNode(node: Node) {
86+
// Execute tasks
87+
for (task in tasks) {
88+
task.handleNode(node)
8089
}
81-
82-
return concept
8390
}
8491

8592
override fun cleanup() {
8693
// Nothing to do
8794
}
8895
}
96+
97+
/**
98+
* Gets concept of type [T] for this [TranslationUnitDeclaration] or creates a new one if it does
99+
* not exist.
100+
*/
101+
inline fun <reified T : Concept> TranslationUnitDeclaration.getConceptOrCreate(
102+
noinline init: ((T) -> Unit)? = null
103+
): T {
104+
var concept = this.conceptNodes.filterIsInstance<T>().singleOrNull()
105+
if (concept == null) {
106+
concept = T::class.constructors.first().call(this)
107+
init?.invoke(concept)
108+
}
109+
110+
return concept
111+
}

cpg-concepts/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/concepts/config/ProvideConfigPass.kt cpg-concepts/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/concepts/config/ProvideConfigTask.kt

+25-17
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
package de.fraunhofer.aisec.cpg.passes.concepts.config
2727

2828
import de.fraunhofer.aisec.cpg.TranslationContext
29+
import de.fraunhofer.aisec.cpg.graph.Component
2930
import de.fraunhofer.aisec.cpg.graph.Node
3031
import de.fraunhofer.aisec.cpg.graph.conceptNodes
3132
import de.fraunhofer.aisec.cpg.graph.concepts.config.Configuration
@@ -43,37 +44,44 @@ import de.fraunhofer.aisec.cpg.graph.operationNodes
4344
import de.fraunhofer.aisec.cpg.graph.translationResult
4445
import de.fraunhofer.aisec.cpg.helpers.Util
4546
import de.fraunhofer.aisec.cpg.passes.concepts.ConceptPass
47+
import de.fraunhofer.aisec.cpg.passes.concepts.ConceptTask
4648
import de.fraunhofer.aisec.cpg.passes.concepts.config.python.stringValues
4749

4850
/**
4951
* This is a generic pass that is responsible for creating [ProvideConfiguration] nodes based on the
5052
* configuration sources found in the graph. It connects a [ConfigurationSource] with a matching
5153
* [Configuration].
5254
*/
53-
class ProvideConfigPass(ctx: TranslationContext) : ConceptPass(ctx) {
54-
override fun handleNode(node: Node, tu: TranslationUnitDeclaration) {
55+
class ProvideConfigTask(target: Component, pass: ConceptPass, ctx: TranslationContext) :
56+
ConceptTask(target, pass, ctx) {
57+
override fun handleNode(node: Node) {
5558
when (node) {
5659
is TranslationUnitDeclaration -> handleTranslationUnit(node)
5760
}
5861
}
5962

6063
private fun handleTranslationUnit(
6164
tu: TranslationUnitDeclaration
62-
): List<ConfigurationOperation> {
63-
// Loop through all configuration sources
64-
return tu.conceptNodes.filterIsInstance<ConfigurationSource>().flatMap { source ->
65-
// Find all LoadConfigurationFile operations that match the INI file name
66-
val loadConfigOps =
67-
tu.translationResult
68-
?.operationNodes
69-
?.filterIsInstance<LoadConfiguration>()
70-
?.filter {
71-
it.fileExpression.stringValues?.contains(source.name.toString()) == true
72-
}
73-
74-
// And create a ProvideConfiguration node for each of them
75-
loadConfigOps?.flatMap { handleConfiguration(source, it.conf, tu, it) } ?: listOf()
76-
}
65+
): List<ConfigurationOperation>? {
66+
return tu.translationResult
67+
?.operationNodes
68+
?.filterIsInstance<LoadConfiguration>()
69+
?.flatMap { loadConfig ->
70+
// Loop through all configuration sources
71+
val sources =
72+
tu.translationResult
73+
?.conceptNodes
74+
?.filterIsInstance<ConfigurationSource>()
75+
?.filter { source ->
76+
loadConfig.fileExpression.stringValues?.contains(
77+
source.name.toString()
78+
) == true
79+
}
80+
81+
// And create a ProvideConfiguration node for each source
82+
sources?.flatMap { handleConfiguration(it, loadConfig.conf, tu, loadConfig) }
83+
?: listOf()
84+
}
7785
}
7886

7987
private fun handleConfiguration(

cpg-concepts/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/concepts/config/ini/IniFileConfigurationSourcePass.kt cpg-concepts/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/concepts/config/ini/IniFileConfigurationSourceTask.kt

+15-10
Original file line numberDiff line numberDiff line change
@@ -26,27 +26,32 @@
2626
package de.fraunhofer.aisec.cpg.passes.concepts.config.ini
2727

2828
import de.fraunhofer.aisec.cpg.TranslationContext
29+
import de.fraunhofer.aisec.cpg.graph.Component
2930
import de.fraunhofer.aisec.cpg.graph.Node
3031
import de.fraunhofer.aisec.cpg.graph.conceptNodes
31-
import de.fraunhofer.aisec.cpg.graph.concepts.config.*
32+
import de.fraunhofer.aisec.cpg.graph.concepts.config.ConfigurationGroupSource
33+
import de.fraunhofer.aisec.cpg.graph.concepts.config.ConfigurationOptionSource
34+
import de.fraunhofer.aisec.cpg.graph.concepts.config.ConfigurationSource
3235
import de.fraunhofer.aisec.cpg.graph.declarations.FieldDeclaration
3336
import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration
3437
import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration
38+
import de.fraunhofer.aisec.cpg.graph.translationUnit
3539
import de.fraunhofer.aisec.cpg.helpers.Util.warnWithFileLocation
36-
import de.fraunhofer.aisec.cpg.passes.ImportResolver
3740
import de.fraunhofer.aisec.cpg.passes.concepts.ConceptPass
38-
import de.fraunhofer.aisec.cpg.passes.concepts.config.ProvideConfigPass
39-
import de.fraunhofer.aisec.cpg.passes.configuration.DependsOn
41+
import de.fraunhofer.aisec.cpg.passes.concepts.ConceptTask
42+
import de.fraunhofer.aisec.cpg.passes.concepts.config.ProvideConfigTask
4043
import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteBefore
41-
import kotlin.collections.singleOrNull
4244

4345
/**
4446
* This pass is responsible for creating [ConfigurationSource] nodes based on the INI file frontend.
4547
*/
46-
@DependsOn(ImportResolver::class)
47-
@ExecuteBefore(ProvideConfigPass::class)
48-
class IniFileConfigurationSourcePass(ctx: TranslationContext) : ConceptPass(ctx) {
49-
override fun handleNode(node: Node, tu: TranslationUnitDeclaration) {
48+
@ExecuteBefore(ProvideConfigTask::class)
49+
class IniFileConfigurationSourceTask(
50+
target: Component,
51+
pass: ConceptPass,
52+
ctx: TranslationContext,
53+
) : ConceptTask(target, pass, ctx) {
54+
override fun handleNode(node: Node) {
5055
// Since we cannot directly depend on the ini frontend, we have to check the language here
5156
// based on the node's language.
5257
if (node.language.name.localName != "IniFileLanguage") {
@@ -55,7 +60,7 @@ class IniFileConfigurationSourcePass(ctx: TranslationContext) : ConceptPass(ctx)
5560

5661
when (node) {
5762
is TranslationUnitDeclaration -> handleTranslationUnit(node)
58-
is RecordDeclaration -> handleRecordDeclaration(node, tu)
63+
is RecordDeclaration -> handleRecordDeclaration(node, node.translationUnit!!)
5964
is FieldDeclaration -> handleFieldDeclaration(node)
6065
}
6166
}

cpg-concepts/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/concepts/config/python/PythonStdLibConfigurationPass.kt cpg-concepts/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/concepts/config/python/PythonStdLibConfigurationTask.kt

+5-18
Original file line numberDiff line numberDiff line change
@@ -27,36 +27,23 @@ package de.fraunhofer.aisec.cpg.passes.concepts.config.python
2727

2828
import de.fraunhofer.aisec.cpg.TranslationContext
2929
import de.fraunhofer.aisec.cpg.analysis.MultiValueEvaluator
30-
import de.fraunhofer.aisec.cpg.graph.Backward
31-
import de.fraunhofer.aisec.cpg.graph.GraphToFollow
32-
import de.fraunhofer.aisec.cpg.graph.Name
33-
import de.fraunhofer.aisec.cpg.graph.Node
30+
import de.fraunhofer.aisec.cpg.graph.*
3431
import de.fraunhofer.aisec.cpg.graph.concepts.config.*
35-
import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration
36-
import de.fraunhofer.aisec.cpg.graph.evaluate
37-
import de.fraunhofer.aisec.cpg.graph.followDFGEdgesUntilHit
38-
import de.fraunhofer.aisec.cpg.graph.followPrevDFG
39-
import de.fraunhofer.aisec.cpg.graph.fqn
40-
import de.fraunhofer.aisec.cpg.graph.implicit
4132
import de.fraunhofer.aisec.cpg.graph.statements.expressions.ConstructExpression
4233
import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberCallExpression
4334
import de.fraunhofer.aisec.cpg.graph.statements.expressions.SubscriptExpression
4435
import de.fraunhofer.aisec.cpg.helpers.Util.warnWithFileLocation
45-
import de.fraunhofer.aisec.cpg.passes.SymbolResolver
4636
import de.fraunhofer.aisec.cpg.passes.concepts.ConceptPass
47-
import de.fraunhofer.aisec.cpg.passes.concepts.config.ProvideConfigPass
48-
import de.fraunhofer.aisec.cpg.passes.configuration.DependsOn
49-
import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteBefore
37+
import de.fraunhofer.aisec.cpg.passes.concepts.ConceptTask
5038

5139
/**
5240
* This pass is responsible for creating [ConfigurationOperation] nodes based on the
5341
* [`configparser`](https://docs.python.org/3/library/configparser.html) module of the Python
5442
* standard library.
5543
*/
56-
@DependsOn(SymbolResolver::class)
57-
@ExecuteBefore(ProvideConfigPass::class)
58-
class PythonStdLibConfigurationPass(ctx: TranslationContext) : ConceptPass(ctx) {
59-
override fun handleNode(node: Node, tu: TranslationUnitDeclaration) {
44+
class PythonStdLibConfigurationTask(target: Component, pass: ConceptPass, ctx: TranslationContext) :
45+
ConceptTask(target, pass, ctx) {
46+
override fun handleNode(node: Node) {
6047
when (node) {
6148
is ConstructExpression -> handleConstructExpression(node)
6249
is MemberCallExpression -> handleMemberCallExpression(node)

0 commit comments

Comments
 (0)