From 698937e95aab7475508e90b0dadefe7e52691666 Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Tue, 17 Dec 2024 20:49:14 +0100 Subject: [PATCH] Adding new Overlay and Concept nodes to the graph. (#1897) --- .../de/fraunhofer/aisec/cpg/graph/Node.kt | 27 ++++-- .../fraunhofer/aisec/cpg/graph/OverlayNode.kt | 47 ++++++++++ .../aisec/cpg/graph/concepts/Concept.kt | 38 ++++++++ .../aisec/cpg/graph/concepts/Operation.kt | 37 ++++++++ .../edges/collections/EdgeSingletonList.kt | 23 ++++- .../aisec/cpg/graph/edges/overlay/Overlay.kt | 88 +++++++++++++++++++ .../aisec/cpg/persistence/TestCommon.kt | 2 + .../aisec/cpg_vis_neo4j/Neo4JTest.kt | 57 ++++++++++-- 8 files changed, 301 insertions(+), 18 deletions(-) create mode 100644 cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/OverlayNode.kt create mode 100644 cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/concepts/Concept.kt create mode 100644 cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/concepts/Operation.kt create mode 100644 cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/overlay/Overlay.kt diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt index 8c07a46ce2..d0ba1a34b3 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt @@ -32,15 +32,16 @@ import de.fraunhofer.aisec.cpg.TranslationContext import de.fraunhofer.aisec.cpg.TypeManager import de.fraunhofer.aisec.cpg.frontends.Handler import de.fraunhofer.aisec.cpg.frontends.Language -import de.fraunhofer.aisec.cpg.graph.declarations.* -import de.fraunhofer.aisec.cpg.graph.edges.* +import de.fraunhofer.aisec.cpg.graph.declarations.MethodDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf -import de.fraunhofer.aisec.cpg.graph.edges.flows.ControlDependences -import de.fraunhofer.aisec.cpg.graph.edges.flows.Dataflows -import de.fraunhofer.aisec.cpg.graph.edges.flows.EvaluationOrders -import de.fraunhofer.aisec.cpg.graph.edges.flows.FullDataflowGranularity -import de.fraunhofer.aisec.cpg.graph.edges.flows.ProgramDependences -import de.fraunhofer.aisec.cpg.graph.scopes.* +import de.fraunhofer.aisec.cpg.graph.edges.flows.* +import de.fraunhofer.aisec.cpg.graph.edges.overlay.* +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping +import de.fraunhofer.aisec.cpg.graph.scopes.GlobalScope +import de.fraunhofer.aisec.cpg.graph.scopes.RecordScope +import de.fraunhofer.aisec.cpg.graph.scopes.Scope import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker import de.fraunhofer.aisec.cpg.helpers.neo4j.LocationConverter import de.fraunhofer.aisec.cpg.helpers.neo4j.NameConverter @@ -52,7 +53,10 @@ import java.util.* import kotlin.uuid.Uuid import org.apache.commons.lang3.builder.ToStringBuilder import org.apache.commons.lang3.builder.ToStringStyle -import org.neo4j.ogm.annotation.* +import org.neo4j.ogm.annotation.GeneratedValue +import org.neo4j.ogm.annotation.Id +import org.neo4j.ogm.annotation.Relationship +import org.neo4j.ogm.annotation.Transient import org.neo4j.ogm.annotation.typeconversion.Convert import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -274,6 +278,11 @@ abstract class Node : */ val additionalProblems: MutableSet = mutableSetOf() + @Relationship(value = "OVERLAY", direction = Relationship.Direction.OUTGOING) + val overlayEdges: Overlays = + Overlays(this, mirrorProperty = OverlayNode::underlyingNodeEdge, outgoing = true) + var overlays by unwrapping(Node::overlayEdges) + /** * If a node should be removed from the graph, just removing it from the AST is not enough (see * issue #60). It will most probably be referenced somewhere via DFG or EOG edges. Thus, if it diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/OverlayNode.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/OverlayNode.kt new file mode 100644 index 0000000000..d187aee975 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/OverlayNode.kt @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph + +import de.fraunhofer.aisec.cpg.graph.edges.overlay.OverlaySingleEdge +import de.fraunhofer.aisec.cpg.graph.edges.unwrapping +import org.neo4j.ogm.annotation.Relationship + +/** + * Represents an extra node added to the CPG. These nodes can live next to the regular nodes, + * typically having shared edges to extend the original graph. + */ +abstract class OverlayNode() : Node() { + @Relationship(value = "OVERLAY", direction = Relationship.Direction.INCOMING) + /** All [OverlayNode]s nodes are connected to an original cpg [Node] by this. */ + val underlyingNodeEdge: OverlaySingleEdge = + OverlaySingleEdge( + this, + of = null, + mirrorProperty = Node::overlayEdges, + outgoing = false, + ) + var underlyingNode by unwrapping(OverlayNode::underlyingNodeEdge) +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/concepts/Concept.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/concepts/Concept.kt new file mode 100644 index 0000000000..8d7aaa42b4 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/concepts/Concept.kt @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.concepts + +import de.fraunhofer.aisec.cpg.graph.OverlayNode + +/** + * Represents a new concept added to the CPG. This is intended for modelling "concepts" like + * logging, files, databases. The relevant operations on this concept are modeled as [Operation]s + * and stored in [ops]. + */ +abstract class Concept() : OverlayNode() { + /** All [Operation]s belonging to this concept. */ + val ops: MutableSet = mutableSetOf() +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/concepts/Operation.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/concepts/Operation.kt new file mode 100644 index 0000000000..014759d647 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/concepts/Operation.kt @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.concepts + +import de.fraunhofer.aisec.cpg.graph.OverlayNode + +/** + * Represents an operation executed on/with a [Concept] (stored in [concept]). This is typically a + * `write` on a file or log object or an `execute` on a database. + */ +abstract class Operation( + /** The [Concept] this operation belongs to. */ + val concept: Concept<*> +) : OverlayNode() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeSingletonList.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeSingletonList.kt index 2b8bab56b2..d61b085630 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeSingletonList.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeSingletonList.kt @@ -36,7 +36,11 @@ import org.neo4j.ogm.annotation.Transient * * Therefore, we need to wrap the edge in a list with a single element. */ -class EdgeSingletonList>( +open class EdgeSingletonList< + NodeType : Node, + NullableNodeType : NodeType?, + EdgeType : Edge +>( override var thisRef: Node, override var init: (Node, NodeType) -> EdgeType, var onChanged: ((old: EdgeType?, new: EdgeType?) -> Unit)? = null, @@ -71,7 +75,15 @@ class EdgeSingletonList): Boolean { @@ -151,6 +163,13 @@ class EdgeSingletonList delegate(): UnwrapDelegate { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/overlay/Overlay.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/overlay/Overlay.kt new file mode 100644 index 0000000000..0a9c4fd7bb --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/overlay/Overlay.kt @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.edges.overlay + +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeSet +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeSingletonList +import de.fraunhofer.aisec.cpg.graph.edges.collections.MirroredEdgeCollection +import kotlin.reflect.KProperty + +/** + * Represents an edge in a graph specifically used for overlay purposes. + * + * @param start The starting node of the edge. + * @param end The ending node of the edge. + * @constructor Constructs an [OverlayEdge] with a specified [start] and [end] node. + * @property labels A predefined set of labels associated with the OverlayEdge. By default, it is + * initialized with the label "OVERLAY". + */ +class OverlayEdge(start: Node, end: Node) : Edge(start, end) { + override var labels: Set = setOf("OVERLAY") +} + +/** + * Represents a single edge in an overlay graph structure, linking nodes with specific properties. + * + * @param thisRef The current node that the edge originates from or is associated with. + * @param of The optional target node of the edge. + * @param mirrorProperty The property representing a mutable collection of mirrored overlay edges. + * @param outgoing A flag indicating whether the edge is outgoing (default is true). + * @constructor Initializes the [OverlaySingleEdge] instance with the provided parameters. + */ +class OverlaySingleEdge( + thisRef: Node, + of: Node?, + override var mirrorProperty: KProperty>, + outgoing: Boolean = true, +) : + EdgeSingletonList( + thisRef = thisRef, + init = ::OverlayEdge, + outgoing = outgoing, + of = of, + ), + MirroredEdgeCollection + +/** + * Represents a collection of overlay edges connected to a specific node. This class is used to + * manage and define relationships between nodes through overlay edges, providing both outgoing and + * incoming edge handling capabilities. + * + * @param thisRef The reference node that the overlays are associated with. + * @param mirrorProperty A reference to a property that mirrors the collection of overlay edges. + * @param outgoing A boolean indicating whether the edges managed by this collection are outgoing. + * @constructor Initializes the [Overlays] object with a reference node, a property for edge + * mirroring, and a direction to specify outgoing or incoming edges. + */ +class Overlays( + thisRef: Node, + override var mirrorProperty: KProperty>, + outgoing: Boolean, +) : + EdgeSet(thisRef = thisRef, init = ::OverlayEdge, outgoing = outgoing), + MirroredEdgeCollection diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/persistence/TestCommon.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/persistence/TestCommon.kt index 8d68203300..f503cdb527 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/persistence/TestCommon.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/persistence/TestCommon.kt @@ -67,6 +67,7 @@ class TestCommon { "DFG", "EOG", "LANGUAGE", + "OVERLAY", "OVERRIDES", "PARAMETERS", "PDG", @@ -91,6 +92,7 @@ class TestCommon { "DFG", "EOG", "LANGUAGE", + "OVERLAY", "PDG", "SCOPE", ), diff --git a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt index 91d2d937d3..40d64eb9c8 100644 --- a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt +++ b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt @@ -26,35 +26,78 @@ package de.fraunhofer.aisec.cpg_vis_neo4j import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.concepts.Concept +import de.fraunhofer.aisec.cpg.graph.concepts.Operation import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal import java.math.BigInteger import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertIs +import kotlin.test.assertNotNull import org.junit.jupiter.api.Tag @Tag("integration") class Neo4JTest { @Test fun testPush() { - val (application, translationResult) = createTranslationResult() + val (application, result) = createTranslationResult() // 22 inferred functions, 1 inferred method, 2 inferred constructors, 11 regular functions - assertEquals(36, translationResult.functions.size) + assertEquals(36, result.functions.size) - application.pushToNeo4j(translationResult) + application.pushToNeo4j(result) } @Test fun testPushVeryLong() { - val (application, translationResult) = createTranslationResult("very_long.cpp") + val (application, result) = createTranslationResult("very_long.cpp") - assertEquals(1, translationResult.variables.size) + assertEquals(1, result.variables.size) - val lit = translationResult.variables["l"]?.initializer + val lit = result.variables["l"]?.initializer assertIs>(lit) assertEquals(BigInteger("10958011617037158669"), lit.value) - application.pushToNeo4j(translationResult) + application.pushToNeo4j(result) + } + + @Test + fun testPushConcepts() { + val (application, result) = createTranslationResult() + + val tu = result.translationUnits.firstOrNull() + assertNotNull(tu) + + val connectCall = result.calls["connect"] + assertNotNull(connectCall) + + abstract class NetworkingOperation( + concept: Concept, + ) : Operation(concept) + class Connect(concept: Concept) : NetworkingOperation(concept) + class Networking() : Concept() + + abstract class FileOperation( + concept: Concept, + ) : Operation(concept) + class FileHandling() : Concept() + + val nw = Networking() + nw.name = Name("Networking") + nw.underlyingNode = tu + + val connect = Connect(concept = nw) + connect.underlyingNode = connectCall + connect.name = Name("connect") + nw.ops += connect + + val f = FileHandling() + f.name = Name("FileHandling") + f.underlyingNode = tu + + assertEquals(setOf(connect), connectCall.overlays) + assertEquals(setOf(nw, f), tu.overlays) + + application.pushToNeo4j(result) } }