From 8ee00be6384fa56f0e6ae671dd319002af3e8c45 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Tue, 10 Dec 2024 19:01:02 +0100 Subject: [PATCH 01/25] Persisting to Neo4J without OGM Work in progress --- cpg-neo4j/build.gradle.kts | 1 + .../de/fraunhofer/aisec/cpg/v2/Persistence.kt | 74 +++++++++++++++++++ .../aisec/cpg/v2/TestPersistence.kt | 39 ++++++++++ gradle/libs.versions.toml | 2 + 4 files changed, 116 insertions(+) create mode 100644 cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt create mode 100644 cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg/v2/TestPersistence.kt diff --git a/cpg-neo4j/build.gradle.kts b/cpg-neo4j/build.gradle.kts index 496f37df4e1..8a626188037 100644 --- a/cpg-neo4j/build.gradle.kts +++ b/cpg-neo4j/build.gradle.kts @@ -50,6 +50,7 @@ publishing { dependencies { // neo4j implementation(libs.bundles.neo4j) + implementation(libs.neo4j.driver) // Command line interface support implementation(libs.picocli) diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt new file mode 100644 index 00000000000..7cb110f6231 --- /dev/null +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.v2 + +import de.fraunhofer.aisec.cpg.TranslationResult +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.nodes +import kotlin.collections.joinToString +import kotlin.reflect.KClass +import org.neo4j.driver.GraphDatabase +import org.neo4j.driver.TransactionContext + +val dbUri = "neo4j://localhost" +val dbUser = "neo4j" +val dbPassword = "password" + +val neo4jSession by lazy { + GraphDatabase.driver(dbUri, org.neo4j.driver.AuthTokens.basic(dbUser, dbPassword)).session() +} + +val labelCache: MutableMap, Set> = mutableMapOf() + +fun TranslationResult.persist() { + neo4jSession.executeWrite { tx -> + with(tx) { + val nodes = this@persist.nodes + for (node in nodes) { + node.persist() + } + } + } +} + +context(TransactionContext) +fun Node.persist() { + val result = + this@TransactionContext.run( + "MERGE (n:${this.labels.joinToString("&")} { name: \$name } ) RETURN elementId(n) AS id", + mapOf("name" to this.name.localName) + ) + val id = result.single()["id"] + println("Created node with id $id") +} + +val Node.labels: Set + get() { + val klazz = this::class + + // Check, if we already computed the labels for this node's class + return labelCache.computeIfAbsent(klazz) { setOf("Node", klazz.simpleName!!) } + } diff --git a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg/v2/TestPersistence.kt b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg/v2/TestPersistence.kt new file mode 100644 index 00000000000..2f9f7daa673 --- /dev/null +++ b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg/v2/TestPersistence.kt @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.v2 + +import de.fraunhofer.aisec.cpg_vis_neo4j.createTranslationResult +import org.junit.jupiter.api.Tag +import org.junit.jupiter.api.Test + +@Tag("integration") +class TestPersistence { + @Test + fun testPersist() { + val result = createTranslationResult() + result.second.persist() + } +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 4a99128145e..9a78d3df12e 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -2,6 +2,7 @@ kotlin = "2.0.20" kotlin19 = "1.9.10" neo4j = "4.0.10" +neo4j5 = "5.27.0" log4j = "2.24.0" spotless = "6.25.0" nexus-publish = "2.0.0" @@ -28,6 +29,7 @@ slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j"} apache-commons-lang3 = { module = "org.apache.commons:commons-lang3", version = "3.17.0"} neo4j-ogm-core = { module = "org.neo4j:neo4j-ogm-core", version.ref = "neo4j"} neo4j-ogm-bolt-driver = { module = "org.neo4j:neo4j-ogm-bolt-driver", version.ref = "neo4j"} +neo4j-driver = { module = "org.neo4j.driver:neo4j-java-driver", version.ref = "neo4j5"} javaparser = { module = "com.github.javaparser:javaparser-symbol-solver-core", version = "3.26.0"} jackson = { module = "com.fasterxml.jackson.module:jackson-module-kotlin", version = "2.18.0"} From 30ee8e408cb74e8fad2123f1452987ea71a5d882 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Tue, 10 Dec 2024 22:00:16 +0100 Subject: [PATCH 02/25] Adding labels to ast edges --- .../fraunhofer/aisec/cpg/TranslationResult.kt | 2 +- .../aisec/cpg/frontends/Language.kt | 2 +- .../fraunhofer/aisec/cpg/graph/Component.kt | 2 +- .../de/fraunhofer/aisec/cpg/graph/Node.kt | 6 +++- .../declarations/EnumConstantDeclaration.kt | 3 +- .../cpg/graph/declarations/EnumDeclaration.kt | 2 +- .../graph/declarations/FunctionDeclaration.kt | 4 +-- .../FunctionTemplateDeclaration.kt | 2 +- .../graph/declarations/IncludeDeclaration.kt | 4 +-- .../graph/declarations/MethodDeclaration.kt | 3 +- .../declarations/NamespaceDeclaration.kt | 4 +-- .../declarations/ParameterDeclaration.kt | 2 +- .../graph/declarations/RecordDeclaration.kt | 12 +++---- .../declarations/RecordTemplateDeclaration.kt | 2 +- .../graph/declarations/TemplateDeclaration.kt | 2 +- .../TranslationUnitDeclaration.kt | 8 ++--- .../graph/declarations/TupleDeclaration.kt | 1 + .../declarations/TypeParameterDeclaration.kt | 2 +- .../graph/declarations/VariableDeclaration.kt | 5 +-- .../fraunhofer/aisec/cpg/graph/edges/Edge.kt | 2 +- .../aisec/cpg/graph/edges/ast/AstEdge.kt | 19 ++++++---- .../cpg/graph/edges/ast/TemplateArgument.kt | 11 ++++-- .../graph/edges/flows/ControlDependence.kt | 2 ++ .../aisec/cpg/graph/edges/flows/Dataflow.kt | 4 +-- .../cpg/graph/statements/AssertStatement.kt | 6 ++-- .../cpg/graph/statements/CaseStatement.kt | 2 +- .../aisec/cpg/graph/statements/CatchClause.kt | 5 +-- .../graph/statements/DeclarationStatement.kt | 2 +- .../aisec/cpg/graph/statements/DoStatement.kt | 3 +- .../cpg/graph/statements/ForEachStatement.kt | 13 +++---- .../cpg/graph/statements/ForStatement.kt | 10 +++--- .../aisec/cpg/graph/statements/IfStatement.kt | 13 ++++--- .../cpg/graph/statements/LabelStatement.kt | 5 +-- .../cpg/graph/statements/LoopStatement.kt | 5 +-- .../cpg/graph/statements/ReturnStatement.kt | 3 +- .../aisec/cpg/graph/statements/Statement.kt | 2 +- .../cpg/graph/statements/SwitchStatement.kt | 10 +++--- .../graph/statements/SynchronizedStatement.kt | 5 +-- .../cpg/graph/statements/ThrowExpression.kt | 5 +-- .../cpg/graph/statements/TryStatement.kt | 13 ++++--- .../cpg/graph/statements/WhileStatement.kt | 5 +-- .../expressions/AssignExpression.kt | 8 +++-- .../statements/expressions/BinaryOperator.kt | 2 ++ .../cpg/graph/statements/expressions/Block.kt | 2 +- .../statements/expressions/CallExpression.kt | 11 +++--- .../statements/expressions/CastExpression.kt | 1 + .../expressions/CollectionComprehension.kt | 6 ++-- .../expressions/ComprehensionExpression.kt | 8 +++-- .../expressions/ConditionalExpression.kt | 7 +++- .../expressions/ConstructExpression.kt | 3 +- .../expressions/DeleteExpression.kt | 2 +- .../statements/expressions/ExpressionList.kt | 2 +- .../expressions/InitializerListExpression.kt | 1 + .../expressions/KeyValueExpression.kt | 6 ++-- .../expressions/LambdaExpression.kt | 6 +++- .../expressions/MemberExpression.kt | 1 + .../expressions/NewArrayExpression.kt | 5 +-- .../statements/expressions/NewExpression.kt | 5 +-- .../statements/expressions/RangeExpression.kt | 6 ++-- .../expressions/SubscriptExpression.kt | 6 +++- .../statements/expressions/UnaryOperator.kt | 1 + .../de/fraunhofer/aisec/cpg/v2/Persistence.kt | 35 ++++++++++++++++--- .../aisec/cpg/v2/TestPersistence.kt | 2 ++ 63 files changed, 221 insertions(+), 118 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt index 05ace1a4dc8..fb0e3907cf0 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt @@ -54,7 +54,7 @@ class TranslationResult( var finalCtx: TranslationContext, ) : Node(), StatisticsHolder { - @Relationship("COMPONENTS") val componentEdges = astEdgesOf() + @Relationship("COMPONENTS") val componentEdges = astEdgesOf(label = "COMPONENTS") /** * Entry points to the CPG: "SoftwareComponent" refer to programs, application, other "bundles" * of software. diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt index 6192a7f1308..9503ba8bc8e 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt @@ -323,7 +323,7 @@ abstract class Language> : Node() { // matches val source = result.source if (this is HasTemplates && source is CallExpression) { - source.templateArgumentEdges = TemplateArguments(source) + source.templateArgumentEdges = TemplateArguments(source, label = "TEMPLATE_ARGUMENTS") val (ok, candidates) = this.handleTemplateFunctionCalls( null, diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Component.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Component.kt index 10f4edaf7dd..73c9d1c47d1 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Component.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Component.kt @@ -43,7 +43,7 @@ import org.neo4j.ogm.annotation.Transient */ open class Component : Node() { @Relationship("TRANSLATION_UNITS") - val translationUnitEdges = astEdgesOf() + val translationUnitEdges = astEdgesOf(label = "TRANSLATION_UNITS") /** All translation units belonging to this application. */ val translationUnits by unwrapping(Component::translationUnitEdges) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt index 8d4ee551c88..d4715d5ac05 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt @@ -48,6 +48,7 @@ import de.fraunhofer.aisec.cpg.passes.* import de.fraunhofer.aisec.cpg.processing.IVisitable import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation import java.util.* +import kotlin.uuid.Uuid import org.apache.commons.lang3.builder.ToStringBuilder import org.apache.commons.lang3.builder.ToStringStyle import org.neo4j.ogm.annotation.* @@ -254,11 +255,14 @@ abstract class Node : /** Required field for object graph mapping. It contains the node id. */ @Id @GeneratedValue var id: Long? = null + /** Will replace [id] */ + var graphId: Uuid? = null + /** Index of the argument if this node is used in a function call or parameter list. */ var argumentIndex = 0 /** List of annotations associated with that node. */ - @Relationship("ANNOTATIONS") var annotationEdges = astEdgesOf() + @Relationship("ANNOTATIONS") var annotationEdges = astEdgesOf(label = "ANNOTATIONS") var annotations by unwrapping(Node::annotationEdges) /** diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumConstantDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumConstantDeclaration.kt index 4ef95234bc6..7786e8fe234 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumConstantDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumConstantDeclaration.kt @@ -36,6 +36,7 @@ import org.neo4j.ogm.annotation.Relationship * explicit initializer value. */ class EnumConstantDeclaration : ValueDeclaration(), HasInitializer { - @Relationship("INITIALIZER") var initializerEdge = astOptionalEdgeOf() + @Relationship("INITIALIZER") + var initializerEdge = astOptionalEdgeOf(label = "INITIALIZER") override var initializer by unwrapping(EnumConstantDeclaration::initializerEdge) } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumDeclaration.kt index c0e912c32f0..b8815215318 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumDeclaration.kt @@ -32,7 +32,7 @@ import org.neo4j.ogm.annotation.Relationship class EnumDeclaration : RecordDeclaration() { @Relationship(value = "ENTRIES", direction = Relationship.Direction.OUTGOING) - var entryEdges = astEdgesOf() + var entryEdges = astEdgesOf(label = "ENTRIES") var entries by unwrapping(EnumDeclaration::entryEdges) override fun toString(): String { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt index f20968b641f..b6072808118 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt @@ -40,13 +40,13 @@ import org.neo4j.ogm.annotation.Relationship /** Represents the declaration or definition of a function. */ open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder, EOGStarterHolder { - @Relationship("BODY") var bodyEdge = astOptionalEdgeOf() + @Relationship("BODY") var bodyEdge = astOptionalEdgeOf(label = "BODY") /** The function body. Usually a [Block]. */ var body by unwrapping(FunctionDeclaration::bodyEdge) /** The list of function parameters. */ @Relationship(value = "PARAMETERS", direction = Relationship.Direction.OUTGOING) - var parameterEdges = astEdgesOf() + var parameterEdges = astEdgesOf(label = "PARAMETERS") /** Virtual property for accessing [parameterEdges] without property edges. */ var parameters by unwrapping(FunctionDeclaration::parameterEdges) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionTemplateDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionTemplateDeclaration.kt index 9b9bcdca326..5c393d18fe0 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionTemplateDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionTemplateDeclaration.kt @@ -39,7 +39,7 @@ class FunctionTemplateDeclaration : TemplateDeclaration() { * expansion pass for each instantiation of the FunctionTemplate there will be a realization */ @Relationship(value = "REALIZATION", direction = Relationship.Direction.OUTGOING) - val realizationEdges = astEdgesOf() + val realizationEdges = astEdgesOf(label = "REALIZATION") val realization by unwrapping(FunctionTemplateDeclaration::realizationEdges) override val realizations: List diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/IncludeDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/IncludeDeclaration.kt index da8e3edb144..994b51cbd8c 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/IncludeDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/IncludeDeclaration.kt @@ -35,11 +35,11 @@ import org.neo4j.ogm.annotation.Relationship /** This declaration represents either an include or an import, depending on the language. */ class IncludeDeclaration : Declaration() { @Relationship(value = "INCLUDES", direction = Relationship.Direction.OUTGOING) - val includeEdges = astEdgesOf() + val includeEdges = astEdgesOf(label = "INCLUDES") val includes by unwrapping(IncludeDeclaration::includeEdges) @Relationship(value = "PROBLEMS", direction = Relationship.Direction.OUTGOING) - val problemEdges = astEdgesOf() + val problemEdges = astEdgesOf(label = "PROBLEMS") val problems by unwrapping(IncludeDeclaration::problemEdges) /** diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/MethodDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/MethodDeclaration.kt index 422e69fcefd..45525f33c1c 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/MethodDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/MethodDeclaration.kt @@ -43,7 +43,8 @@ open class MethodDeclaration : FunctionDeclaration() { */ open var recordDeclaration: RecordDeclaration? = null - @Relationship("RECEIVER") var receiverEdge = astOptionalEdgeOf() + @Relationship("RECEIVER") + var receiverEdge = astOptionalEdgeOf(label = "RECEIVER") /** * The receiver variable of this method. In most cases, this variable is called `this`, but in * some languages, it is `self` (e.g. in Rust or Python) or can be freely named (e.g. in diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt index c2e9927eb5e..6337a682a0b 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt @@ -48,12 +48,12 @@ class NamespaceDeclaration : Declaration(), DeclarationHolder, StatementHolder, * Edges to nested namespaces, records, functions, fields etc. contained in the current * namespace. */ - val declarationEdges = astEdgesOf() + val declarationEdges = astEdgesOf(label = "DECLARATIONS") override val declarations by unwrapping(NamespaceDeclaration::declarationEdges) /** The list of statements. */ @Relationship(value = "STATEMENTS", direction = Relationship.Direction.OUTGOING) - override var statementEdges = astEdgesOf() + override var statementEdges = astEdgesOf(label = "STATEMENTS") /** * In some languages, there is a relationship between paths / directories and the package diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ParameterDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ParameterDeclaration.kt index cd820498d05..9b0fb81753d 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ParameterDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ParameterDeclaration.kt @@ -37,7 +37,7 @@ class ParameterDeclaration : ValueDeclaration(), HasDefault { var isVariadic = false @Relationship(value = "DEFAULT", direction = Relationship.Direction.OUTGOING) - var defaultValueEdge = astOptionalEdgeOf() + var defaultValueEdge = astOptionalEdgeOf(label = "DEFAULT") private var defaultValue by unwrapping(ParameterDeclaration::defaultValueEdge) var modifiers: List = mutableListOf() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt index 86a9ceef33f..27f5fa01b7b 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt @@ -44,28 +44,28 @@ open class RecordDeclaration : var kind: String? = null @Relationship(value = "FIELDS", direction = Relationship.Direction.OUTGOING) - var fieldEdges = astEdgesOf() + var fieldEdges = astEdgesOf(label = "FIELDS") var fields by unwrapping(RecordDeclaration::fieldEdges) @Relationship(value = "METHODS", direction = Relationship.Direction.OUTGOING) - var methodEdges = astEdgesOf() + var methodEdges = astEdgesOf(label = "METHODS") var methods by unwrapping(RecordDeclaration::methodEdges) @Relationship(value = "CONSTRUCTORS", direction = Relationship.Direction.OUTGOING) - var constructorEdges = astEdgesOf() + var constructorEdges = astEdgesOf(label = "CONSTRUCTORS") var constructors by unwrapping(RecordDeclaration::constructorEdges) @Relationship(value = "RECORDS", direction = Relationship.Direction.OUTGOING) - var recordEdges = astEdgesOf() + var recordEdges = astEdgesOf(label = "RECORDS") var records by unwrapping(RecordDeclaration::recordEdges) @Relationship(value = "TEMPLATES", direction = Relationship.Direction.OUTGOING) - var templateEdges = astEdgesOf() + var templateEdges = astEdgesOf(label = "TEMPLATES") var templates by unwrapping(RecordDeclaration::templateEdges) /** The list of statements. */ @Relationship(value = "STATEMENTS", direction = Relationship.Direction.OUTGOING) - override var statementEdges = astEdgesOf() + override var statementEdges = astEdgesOf(label = "STATEMENTS") override var statements by unwrapping(RecordDeclaration::statementEdges) @Transient var superClasses: MutableList = ArrayList() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordTemplateDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordTemplateDeclaration.kt index 56e4b316454..028ef222e24 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordTemplateDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordTemplateDeclaration.kt @@ -39,7 +39,7 @@ class RecordTemplateDeclaration : TemplateDeclaration() { * expansion pass for each instantiation of the ClassTemplate there will be a realization */ @Relationship(value = "REALIZATION", direction = Relationship.Direction.OUTGOING) - val realizationEdges = astEdgesOf() + val realizationEdges = astEdgesOf(label = "REALIZATION") override val realizations by unwrapping(RecordTemplateDeclaration::realizationEdges) override fun addDeclaration(declaration: Declaration) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TemplateDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TemplateDeclaration.kt index 2b53dc4efa9..4048a04da41 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TemplateDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TemplateDeclaration.kt @@ -51,7 +51,7 @@ abstract class TemplateDeclaration : Declaration(), DeclarationHolder { /** Parameters the Template requires for instantiation */ @Relationship(value = "PARAMETERS", direction = Relationship.Direction.OUTGOING) - var parameterEdges = astEdgesOf() + var parameterEdges = astEdgesOf(label = "PARAMETERS") val parameters by unwrapping(TemplateDeclaration::parameterEdges) val parametersWithDefaults: List diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt index 99b6f5ea162..afd80853394 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt @@ -39,22 +39,22 @@ class TranslationUnitDeclaration : Declaration(), DeclarationHolder, StatementHolder, EOGStarterHolder { /** A list of declarations within this unit. */ @Relationship(value = "DECLARATIONS", direction = Relationship.Direction.OUTGOING) - val declarationEdges = astEdgesOf() + val declarationEdges = astEdgesOf(label = "DECLARATIONS") override val declarations by unwrapping(TranslationUnitDeclaration::declarationEdges) /** A list of includes within this unit. */ @Relationship(value = "INCLUDES", direction = Relationship.Direction.OUTGOING) - val includeEdges = astEdgesOf() + val includeEdges = astEdgesOf(label = "INCLUDES") val includes by unwrapping(TranslationUnitDeclaration::includeEdges) /** A list of namespaces within this unit. */ @Relationship(value = "NAMESPACES", direction = Relationship.Direction.OUTGOING) - val namespaceEdges = astEdgesOf() + val namespaceEdges = astEdgesOf(label = "NAMESPACES") val namespaces by unwrapping(TranslationUnitDeclaration::namespaceEdges) /** The list of statements. */ @Relationship(value = "STATEMENTS", direction = Relationship.Direction.OUTGOING) - override var statementEdges = astEdgesOf() + override var statementEdges = astEdgesOf(label = "STATEMENTS") override var statements by unwrapping(TranslationUnitDeclaration::statementEdges) override fun addDeclaration(declaration: Declaration) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TupleDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TupleDeclaration.kt index acd7fe230f6..4f70b631e3b 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TupleDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TupleDeclaration.kt @@ -65,6 +65,7 @@ class TupleDeclaration : VariableDeclaration() { /** The list of elements in this tuple. */ var elementEdges = astEdgesOf( + label = "ELEMENTS", onAdd = { registerTypeObserver(it.end) }, onRemove = { unregisterTypeObserver(it.end) } ) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TypeParameterDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TypeParameterDeclaration.kt index 2f061836a24..cd558339e99 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TypeParameterDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TypeParameterDeclaration.kt @@ -35,7 +35,7 @@ import org.neo4j.ogm.annotation.Relationship /** A declaration of a type template parameter */ class TypeParameterDeclaration : ValueDeclaration(), HasDefault { @Relationship(value = "DEFAULT", direction = Relationship.Direction.OUTGOING) - var defaultEdge = astOptionalEdgeOf() + var defaultEdge = astOptionalEdgeOf(label = "DEFAULT") /** TemplateParameters can define a default for the type parameter. */ override var default by unwrapping(TypeParameterDeclaration::defaultEdge) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt index 83fecda3e0d..cbaa05e56d5 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt @@ -48,7 +48,7 @@ open class VariableDeclaration : ValueDeclaration(), HasInitializer, HasType.Typ * the [ConstructExpression] is created. */ @Relationship(value = "TEMPLATE_PARAMETERS", direction = Relationship.Direction.OUTGOING) - var templateParameterEdges = astEdgesOf() + var templateParameterEdges = astEdgesOf(label = "TEMPLATE_PARAMETERS") var templateParameters by unwrapping(VariableDeclaration::templateParameterEdges) /** Determines if this is a global variable. */ @@ -69,13 +69,14 @@ open class VariableDeclaration : ValueDeclaration(), HasInitializer, HasType.Typ @Relationship("INITIALIZER") var initializerEdge = astOptionalEdgeOf( + label = "INITIALIZER", onChanged = { old, new -> val value = new?.end exchangeTypeObserver(old, new) if (value is Reference) { value.resolutionHelper = this } - } + }, ) /** The (optional) initializer of the declaration. */ override var initializer by unwrapping(VariableDeclaration::initializerEdge) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt index 1bb365dd633..67428529070 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt @@ -71,7 +71,7 @@ abstract class Edge : Persistable, Cloneable { end = edge.end } - @Transient open val label: String = "EDGE" + @Transient open var label: String = "EDGE" /** * The index of this node, if it is stored in an diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt index ebdc6b69e56..cd2d2f9fafd 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt @@ -33,18 +33,20 @@ import org.neo4j.ogm.annotation.* /** This property edge describes a parent/child relationship in the Abstract Syntax Tree (AST). */ @RelationshipEntity -open class AstEdge(start: Node, end: T) : Edge(start, end) { +open class AstEdge(start: Node, end: T, label: String = "AST") : Edge(start, end) { init { end.astParent = start + this.label = label } } /** Creates an [AstEdges] container starting from this node. */ fun Node.astEdgesOf( + label: String = "AST", onAdd: ((AstEdge) -> Unit)? = null, onRemove: ((AstEdge) -> Unit)? = null, ): AstEdges> { - return AstEdges(thisRef = this, onAdd = onAdd, onRemove = onRemove) + return AstEdges(thisRef = this, label = label, onAdd = onAdd, onRemove = onRemove) } /** @@ -52,11 +54,12 @@ fun Node.astEdgesOf( * container). */ fun Node.astOptionalEdgeOf( - onChanged: ((old: AstEdge?, new: AstEdge?) -> Unit)? = null, + label: String = "AST", + onChanged: ((old: AstEdge?, new: AstEdge?) -> Unit)? = null ): EdgeSingletonList> { return EdgeSingletonList( thisRef = this, - init = ::AstEdge, + init = { start, end -> AstEdge(start, end, label = label) }, outgoing = true, onChanged = onChanged, of = null @@ -68,11 +71,12 @@ fun Node.astOptionalEdgeOf( */ fun Node.astEdgeOf( of: NodeType, + label: String, onChanged: ((old: AstEdge?, new: AstEdge?) -> Unit)? = null, ): EdgeSingletonList> { return EdgeSingletonList( thisRef = this, - init = ::AstEdge, + init = { start, end -> AstEdge(start, end, label = label) }, outgoing = true, onChanged = onChanged, of = of @@ -82,12 +86,13 @@ fun Node.astEdgeOf( /** This property edge list describes elements that are AST children of a node. */ open class AstEdges>( thisRef: Node, + label: String = "AST", onAdd: ((PropertyEdgeType) -> Unit)? = null, onRemove: ((PropertyEdgeType) -> Unit)? = null, @Suppress("UNCHECKED_CAST") init: (start: Node, end: NodeType) -> PropertyEdgeType = { start, end -> - AstEdge(start, end) as PropertyEdgeType - } + AstEdge(start, end, label = label) as PropertyEdgeType + }, ) : EdgeList( thisRef = thisRef, diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/TemplateArgument.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/TemplateArgument.kt index b67f7711eaf..b06c3bdfd19 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/TemplateArgument.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/TemplateArgument.kt @@ -33,9 +33,14 @@ import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression class TemplateArgument( start: Node, end: NodeType, + label: String, var instantiation: TemplateInitialization? = TemplateInitialization.EXPLICIT, -) : AstEdge(start, end) +) : AstEdge(start, end, label) /** A container for [TemplateArgument] edges. */ -class TemplateArguments(thisRef: Node) : - AstEdges>(thisRef, init = ::TemplateArgument) +class TemplateArguments(thisRef: Node, label: String) : + AstEdges>( + thisRef, + label, + init = { start, end -> TemplateArgument(start, end, label = label) } + ) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ControlDependence.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ControlDependence.kt index e374adfcefd..3591e0eaf56 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ControlDependence.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ControlDependence.kt @@ -49,6 +49,8 @@ class ControlDependence( dependence = DependenceType.CONTROL } + override var label: String = "CDG" + override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is ControlDependence) return false diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Dataflow.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Dataflow.kt index 900eedc09b0..4ce9ff4ebdb 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Dataflow.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Dataflow.kt @@ -93,7 +93,7 @@ open class Dataflow( @JsonIgnore var granularity: Granularity = default() ) : Edge(start, end) { - override val label: String = "DFG" + override var label: String = "DFG" override fun equals(other: Any?): Boolean { if (this === other) return true @@ -134,7 +134,7 @@ class ContextSensitiveDataflow( val callingContext: CallingContext ) : Dataflow(start, end, granularity) { - override val label: String = "DFG" + override var label: String = "DFG" override fun equals(other: Any?): Boolean { if (this === other) return true diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/AssertStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/AssertStatement.kt index bc9910b4295..2974aa06175 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/AssertStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/AssertStatement.kt @@ -33,11 +33,13 @@ import org.neo4j.ogm.annotation.Relationship /** Represents an assert statement */ class AssertStatement : Statement() { - @Relationship(value = "CONDITION") var conditionEdge = astOptionalEdgeOf() + @Relationship(value = "CONDITION") + var conditionEdge = astOptionalEdgeOf(label = "CONDITION") /** The condition to be evaluated. */ var condition by unwrapping(AssertStatement::conditionEdge) - @Relationship(value = "MESSAGE") var messageEdge = astOptionalEdgeOf() + @Relationship(value = "MESSAGE") + var messageEdge = astOptionalEdgeOf(label = "MESSAGE") /** The *optional* message that is shown, if the assert is evaluated as true */ var message by unwrapping(AssertStatement::messageEdge) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CaseStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CaseStatement.kt index e0dfe6f36f9..1c65983ddb4 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CaseStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CaseStatement.kt @@ -39,7 +39,7 @@ import org.neo4j.ogm.annotation.Relationship */ class CaseStatement : Statement() { @Relationship(value = "CASE_EXPRESSION") - var caseExpressionEdge = astOptionalEdgeOf() + var caseExpressionEdge = astOptionalEdgeOf(label = "CASE_EXPRESSION") /** * Primitive side effect free statement that has to match with the evaluated selector in diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CatchClause.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CatchClause.kt index 5f317eff4c0..025de94b67a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CatchClause.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CatchClause.kt @@ -36,11 +36,12 @@ import java.util.Objects import org.neo4j.ogm.annotation.Relationship class CatchClause : Statement(), BranchingNode, EOGStarterHolder { - @Relationship(value = "PARAMETER") var parameterEdge = astOptionalEdgeOf() + @Relationship(value = "PARAMETER") + var parameterEdge = astOptionalEdgeOf(label = "PARAMETER") var parameter by unwrapping(CatchClause::parameterEdge) - @Relationship(value = "BODY") var bodyEdge = astOptionalEdgeOf() + @Relationship(value = "BODY") var bodyEdge = astOptionalEdgeOf(label = "BODY") var body by unwrapping(CatchClause::bodyEdge) override val branchedBy: Node? diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.kt index 31285ccc599..dd16600207b 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.kt @@ -45,7 +45,7 @@ open class DeclarationStatement : Statement() { * it only contains a single [Declaration]. */ @Relationship(value = "DECLARATIONS", direction = Relationship.Direction.OUTGOING) - var declarationEdges = astEdgesOf() + var declarationEdges = astEdgesOf(label = "DECLARATIONS") override var declarations by unwrapping(DeclarationStatement::declarationEdges) var singleDeclaration: Declaration? diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DoStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DoStatement.kt index 256546f5519..24711e2e2ea 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DoStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DoStatement.kt @@ -39,7 +39,8 @@ import org.neo4j.ogm.annotation.Relationship * a [Block], is executed and re-executed if the [condition] evaluates to true. */ class DoStatement : LoopStatement(), ArgumentHolder { - @Relationship("CONDITION") var conditionEdge = astOptionalEdgeOf() + @Relationship("CONDITION") + var conditionEdge = astOptionalEdgeOf(label = "CONDITION") /** * The loop condition that is evaluated after the loop statement and may trigger reevaluation. */ diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForEachStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForEachStatement.kt index 029aef766f8..2f6b9d3f6f7 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForEachStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForEachStatement.kt @@ -45,6 +45,7 @@ class ForEachStatement : LoopStatement(), BranchingNode, StatementHolder { @Relationship("VARIABLE") var variableEdge = astOptionalEdgeOf( + label = "VARIABLE", onChanged = { _, new -> val end = new?.end if (end is Reference) { @@ -59,7 +60,7 @@ class ForEachStatement : LoopStatement(), BranchingNode, StatementHolder { */ var variable by unwrapping(ForEachStatement::variableEdge) - @Relationship("ITERABLE") var iterableEdge = astOptionalEdgeOf() + @Relationship("ITERABLE") var iterableEdge = astOptionalEdgeOf(label = "ITERABLE") /** This field contains the iteration subject of the loop. */ var iterable by unwrapping(ForEachStatement::iterableEdge) @@ -68,11 +69,11 @@ class ForEachStatement : LoopStatement(), BranchingNode, StatementHolder { override var statementEdges: AstEdges> get() { - val statements = astEdgesOf() - variable?.let { statements.add(AstEdge(this, it)) } - iterable?.let { statements.add(AstEdge(this, it)) } - statement?.let { statements.add(AstEdge(this, it)) } - elseStatement?.let { statements.add(AstEdge(this, it)) } + val statements = astEdgesOf(label = "STATEMENTS") + statements += variableEdge + statements += iterableEdge + statements += statementEdge + statements += elseStatementEdge return statements } set(_) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForStatement.kt index 482c7a32c76..4e59de9266a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForStatement.kt @@ -42,17 +42,19 @@ import org.neo4j.ogm.annotation.Relationship class ForStatement : LoopStatement(), BranchingNode { @Relationship("INITIALIZER_STATEMENT") - var initializerStatementEdge = astOptionalEdgeOf() + var initializerStatementEdge = astOptionalEdgeOf(label = "INITIALIZER_STATEMENT") var initializerStatement by unwrapping(ForStatement::initializerStatementEdge) @Relationship("CONDITION_DECLARATION") - var conditionDeclarationEdge = astOptionalEdgeOf() + var conditionDeclarationEdge = astOptionalEdgeOf(label = "CONDITION_DECLARATION") var conditionDeclaration by unwrapping(ForStatement::conditionDeclarationEdge) - @Relationship("CONDITION") var conditionEdge = astOptionalEdgeOf() + @Relationship("CONDITION") + var conditionEdge = astOptionalEdgeOf(label = "CONDITION") var condition by unwrapping(ForStatement::conditionEdge) - @Relationship("ITERATION_STATEMENT") var iterationStatementEdge = astOptionalEdgeOf() + @Relationship("ITERATION_STATEMENT") + var iterationStatementEdge = astOptionalEdgeOf(label = "ITERATION_STATEMENT") var iterationStatement by unwrapping(ForStatement::iterationStatementEdge) override val branchedBy: Node? diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/IfStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/IfStatement.kt index 346d78b533f..dfdc2f9d05e 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/IfStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/IfStatement.kt @@ -40,16 +40,17 @@ import org.neo4j.ogm.annotation.Relationship /** Represents a condition control flow statement, usually indicating by `If`. */ class IfStatement : Statement(), BranchingNode, ArgumentHolder { @Relationship(value = "INITIALIZER_STATEMENT") - var initializerStatementEdge = astOptionalEdgeOf() + var initializerStatementEdge = astOptionalEdgeOf(label = "INITIALIZER_STATEMENT") /** C++ initializer statement. */ var initializerStatement by unwrapping(IfStatement::initializerStatementEdge) @Relationship(value = "CONDITION_DECLARATION") - var conditionDeclarationEdge = astOptionalEdgeOf() + var conditionDeclarationEdge = astOptionalEdgeOf(label = "CONDITION_DECLARATION") /** C++ alternative to the condition. */ var conditionDeclaration by unwrapping(IfStatement::conditionDeclarationEdge) - @Relationship(value = "CONDITION") var conditionEdge = astOptionalEdgeOf() + @Relationship(value = "CONDITION") + var conditionEdge = astOptionalEdgeOf(label = "CONDITION") /** The condition to be evaluated. */ var condition by unwrapping(IfStatement::conditionEdge) @@ -59,11 +60,13 @@ class IfStatement : Statement(), BranchingNode, ArgumentHolder { /** C++ constexpr construct. */ var isConstExpression = false - @Relationship(value = "THEN_STATEMENT") var thenStatementEdge = astOptionalEdgeOf() + @Relationship(value = "THEN_STATEMENT") + var thenStatementEdge = astOptionalEdgeOf(label = "THEN_STATEMENT") /** The statement that is executed, if the condition is evaluated as true. Usually a [Block]. */ var thenStatement by unwrapping(IfStatement::thenStatementEdge) - @Relationship(value = "ELSE_STATEMENT") var elseStatementEdge = astOptionalEdgeOf() + @Relationship(value = "ELSE_STATEMENT") + var elseStatementEdge = astOptionalEdgeOf(label = "ELSE_STATEMENT") /** * The statement that is executed, if the condition is evaluated as false. Usually a [Block]. */ diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/LabelStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/LabelStatement.kt index 88eddde61cb..0eff2e9b7a7 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/LabelStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/LabelStatement.kt @@ -40,7 +40,8 @@ import org.neo4j.ogm.annotation.Relationship * breaks (Java) or goto(C++). */ class LabelStatement : Statement(), StatementHolder { - @Relationship(value = "SUB_STATEMENT") var subStatementEdge = astOptionalEdgeOf() + @Relationship(value = "SUB_STATEMENT") + var subStatementEdge = astOptionalEdgeOf(label = "SUB_STATEMENT") /** Statement that the label is attached to. Can be a simple or compound statement. */ var subStatement by unwrapping(LabelStatement::subStatementEdge) @@ -58,7 +59,7 @@ class LabelStatement : Statement(), StatementHolder { override var statementEdges: AstEdges> get() { - var list = astEdgesOf() + var list = astEdgesOf(label = "STATEMENTS") subStatement?.let { list.resetTo(listOf(it)) } return list } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/LoopStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/LoopStatement.kt index 5db8239930a..13b551faeb6 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/LoopStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/LoopStatement.kt @@ -43,7 +43,7 @@ import org.neo4j.ogm.annotation.Relationship */ abstract class LoopStatement : Statement() { - @Relationship("STATEMENT") var statementEdge = astOptionalEdgeOf() + @Relationship("STATEMENT") var statementEdge = astOptionalEdgeOf(label = "STATEMENT") /** This field contains the body of the loop, e.g. a [Block] or single [Statement]. */ var statement by unwrapping(LoopStatement::statementEdge) @@ -54,7 +54,8 @@ abstract class LoopStatement : Statement() { * `else`-Statement at loop level. E.g. in Python the [elseStatement] is executed when the loop * was not left through a break. */ - @Relationship(value = "ELSE_STATEMENT") var elseStatementEdge = astOptionalEdgeOf() + @Relationship(value = "ELSE_STATEMENT") + var elseStatementEdge = astOptionalEdgeOf(label = "ELSE_STATEMENT") var elseStatement by unwrapping(LoopStatement::elseStatementEdge) override fun toString() = diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ReturnStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ReturnStatement.kt index 420db4def05..f5ba10c00ed 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ReturnStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ReturnStatement.kt @@ -35,7 +35,8 @@ import org.neo4j.ogm.annotation.Relationship /** Represents a statement that returns out of the current function. */ class ReturnStatement : Statement(), ArgumentHolder { - @Relationship(value = "RETURN_VALUES") var returnValueEdges = astEdgesOf() + @Relationship(value = "RETURN_VALUES") + var returnValueEdges = astEdgesOf(label = "RETURN_VALUES") /** The expression whose value will be returned. */ var returnValues by unwrapping(ReturnStatement::returnValueEdges) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/Statement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/Statement.kt index f006d9c132e..129c3ddb2d8 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/Statement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/Statement.kt @@ -52,7 +52,7 @@ abstract class Statement : Node(), DeclarationHolder { * TODO: This is actually an AST node just for a subset of nodes, i.e. initializers in for-loops */ @Relationship(value = "LOCALS", direction = Relationship.Direction.OUTGOING) - var localEdges = astEdgesOf() + var localEdges = astEdgesOf(label = "LOCALS") /** Virtual property to access [localEdges] without property edges. */ var locals by unwrapping(Statement::localEdges) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SwitchStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SwitchStatement.kt index 9d955a50575..3277d36cb19 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SwitchStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SwitchStatement.kt @@ -40,21 +40,23 @@ import org.neo4j.ogm.annotation.Relationship * handled properly. */ class SwitchStatement : Statement(), BranchingNode { - @Relationship(value = "SELECTOR") var selectorEdge = astOptionalEdgeOf() + @Relationship(value = "SELECTOR") + var selectorEdge = astOptionalEdgeOf(label = "SELECTOR") /** Selector that determines the case/default statement of the subsequent execution */ var selector by unwrapping(SwitchStatement::selectorEdge) @Relationship(value = "INITIALIZER_STATEMENT") - var initializerStatementEdge = astOptionalEdgeOf() + var initializerStatementEdge = astOptionalEdgeOf(label = "INITIALIZER_STATEMENT") /** C++ can have an initializer statement in a switch */ var initializerStatement by unwrapping(SwitchStatement::initializerStatementEdge) @Relationship(value = "SELECTOR_DECLARATION") - var selectorDeclarationEdge = astOptionalEdgeOf() + var selectorDeclarationEdge = astOptionalEdgeOf(label = "SELECTOR_DECLARATION") /** C++ allows to use a declaration instead of an expression as selector */ var selectorDeclaration by unwrapping(SwitchStatement::selectorDeclarationEdge) - @Relationship(value = "STATEMENT") var statementEdge = astOptionalEdgeOf() + @Relationship(value = "STATEMENT") + var statementEdge = astOptionalEdgeOf(label = "STATEMENT") /** * The compound statement that contains break/default statements with regular statements on the * same hierarchy diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SynchronizedStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SynchronizedStatement.kt index ac955b24ce6..0608e511552 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SynchronizedStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SynchronizedStatement.kt @@ -33,10 +33,11 @@ import java.util.Objects import org.neo4j.ogm.annotation.Relationship class SynchronizedStatement : Statement() { - @Relationship(value = "EXPRESSION") var expressionEdge = astOptionalEdgeOf() + @Relationship(value = "EXPRESSION") + var expressionEdge = astOptionalEdgeOf(label = "EXPRESSION") var expression by unwrapping(SynchronizedStatement::expressionEdge) - @Relationship(value = "BLOCK") var blockEdge = astOptionalEdgeOf() + @Relationship(value = "BLOCK") var blockEdge = astOptionalEdgeOf(label = "BLOCK") var block by unwrapping(SynchronizedStatement::blockEdge) override fun equals(other: Any?): Boolean { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ThrowExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ThrowExpression.kt index 0f8b86cb32a..59d245f79b3 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ThrowExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ThrowExpression.kt @@ -37,7 +37,8 @@ import org.neo4j.ogm.annotation.Relationship class ThrowExpression : Expression(), ArgumentHolder { /** The exception object to be raised. */ - @Relationship(value = "EXCEPTION") var exceptionEdge = astOptionalEdgeOf() + @Relationship(value = "EXCEPTION") + var exceptionEdge = astOptionalEdgeOf(label = "EXCEPTION") var exception by unwrapping(ThrowExpression::exceptionEdge) /** @@ -45,7 +46,7 @@ class ThrowExpression : Expression(), ArgumentHolder { * was raised while handling another exception. */ @Relationship(value = "PARENT_EXCEPTION") - var parentExceptionEdge = astOptionalEdgeOf() + var parentExceptionEdge = astOptionalEdgeOf(label = "PARENT_EXCEPTION") var parentException by unwrapping(ThrowExpression::parentExceptionEdge) override fun addArgument(expression: Expression) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/TryStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/TryStatement.kt index 1f887fa2075..cd96ca44d31 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/TryStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/TryStatement.kt @@ -42,14 +42,15 @@ class TryStatement : Statement() { * enter the [tryBlock]. */ @Relationship(value = "RESOURCES", direction = Relationship.Direction.OUTGOING) - var resourceEdges = astEdgesOf() + var resourceEdges = astEdgesOf(label = "RESOURCES") var resources by unwrapping(TryStatement::resourceEdges) /** * This represents a block whose statements can throw exceptions which are handled by the * [catchClauses]. */ - @Relationship(value = "TRY_BLOCK") var tryBlockEdge = astOptionalEdgeOf() + @Relationship(value = "TRY_BLOCK") + var tryBlockEdge = astOptionalEdgeOf(label = "TRY_BLOCK") var tryBlock by unwrapping(TryStatement::tryBlockEdge) /** @@ -57,7 +58,8 @@ class TryStatement : Statement() { * exceptions. Note that any exception thrown in this block is no longer caught by the * [catchClauses]. */ - @Relationship(value = "ELSE_BLOCK") var elseBlockEdge = astOptionalEdgeOf() + @Relationship(value = "ELSE_BLOCK") + var elseBlockEdge = astOptionalEdgeOf(label = "ELSE_BLOCK") var elseBlock by unwrapping(TryStatement::elseBlockEdge) /** @@ -65,7 +67,8 @@ class TryStatement : Statement() { * or one of the [catchClauses]. Note that any exception thrown in this block is no longer * caught by the [catchClauses]. */ - @Relationship(value = "FINALLY_BLOCK") var finallyBlockEdge = astOptionalEdgeOf() + @Relationship(value = "FINALLY_BLOCK") + var finallyBlockEdge = astOptionalEdgeOf(label = "FINALLY_BLOCK") var finallyBlock by unwrapping(TryStatement::finallyBlockEdge) /** @@ -74,7 +77,7 @@ class TryStatement : Statement() { * exists. */ @Relationship(value = "CATCH_CLAUSES", direction = Relationship.Direction.OUTGOING) - var catchClauseEdges = astEdgesOf() + var catchClauseEdges = astEdgesOf(label = "CATCH_CLAUSES") var catchClauses by unwrapping(TryStatement::catchClauseEdges) override fun equals(other: Any?): Boolean { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/WhileStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/WhileStatement.kt index 5c3bfb69b41..ffb9d38cc6e 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/WhileStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/WhileStatement.kt @@ -42,11 +42,12 @@ import org.neo4j.ogm.annotation.Relationship */ class WhileStatement : LoopStatement(), BranchingNode, ArgumentHolder { @Relationship(value = "CONDITION_DECLARATION") - var conditionDeclarationEdge = astOptionalEdgeOf() + var conditionDeclarationEdge = astOptionalEdgeOf(label = "CONDITION_DECLARATION") /** C++ allows defining a declaration instead of a pure logical expression as condition */ var conditionDeclaration by unwrapping(WhileStatement::conditionDeclarationEdge) - @Relationship(value = "CONDITION") var conditionEdge = astOptionalEdgeOf() + @Relationship(value = "CONDITION") + var conditionEdge = astOptionalEdgeOf(label = "CONDITION") /** The condition that decides if the block is executed. */ var condition by unwrapping(WhileStatement::conditionEdge) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt index 966a9135457..c1f335c565b 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt @@ -63,6 +63,7 @@ class AssignExpression : @Relationship("LHS") var lhsEdges = astEdgesOf( + label = "LHS", onAdd = { var end = it.end var base = (end as? MemberExpression)?.base as? MemberExpression @@ -81,12 +82,12 @@ class AssignExpression : var lhs by unwrapping(AssignExpression::lhsEdges) @Relationship("RHS") - /** The expressions on the right-hand side. */ var rhsEdges = astEdgesOf( + label = "RHS", onAdd = { it.end.registerTypeObserver(this) }, - onRemove = { it.end.unregisterTypeObserver(this) }, + onRemove = { it.end.unregisterTypeObserver(this) } ) var rhs by unwrapping(AssignExpression::rhsEdges) @@ -125,7 +126,8 @@ class AssignExpression : return operatorCode in (language?.simpleAssignmentOperators ?: setOf()) } - @Relationship("DECLARATIONS") var declarationEdges = astEdgesOf() + @Relationship("DECLARATIONS") + var declarationEdges = astEdgesOf(label = "DECLARATIONS") /** * Some languages, such as Go explicitly allow the definition / declaration of variables in the * assignment (known as a "short assignment"). Some languages, such as Python even implicitly diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/BinaryOperator.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/BinaryOperator.kt index 06cce9f4221..63e1ccec0d4 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/BinaryOperator.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/BinaryOperator.kt @@ -50,6 +50,7 @@ open class BinaryOperator : var lhsEdge = astEdgeOf( of = ProblemExpression("could not parse lhs"), + label = "LHS", onChanged = ::exchangeTypeObserver ) var lhs by unwrapping(BinaryOperator::lhsEdge) @@ -59,6 +60,7 @@ open class BinaryOperator : var rhsEdge = astEdgeOf( of = ProblemExpression("could not parse rhs"), + label = "RHS", onChanged = ::exchangeTypeObserver ) var rhs by unwrapping(BinaryOperator::rhsEdge) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Block.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Block.kt index 0677c750870..16ad6ad0f33 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Block.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Block.kt @@ -42,7 +42,7 @@ import org.neo4j.ogm.annotation.Relationship open class Block : Expression(), StatementHolder { /** The list of statements. */ @Relationship(value = "STATEMENTS", direction = Relationship.Direction.OUTGOING) - override var statementEdges = astEdgesOf() + override var statementEdges = astEdgesOf(label = "STATEMENTS") override var statements by unwrapping(Block::statementEdges) /** * This variable helps to differentiate between static and non-static initializer blocks. Static diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt index cf9056e16ae..218ae9d250d 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt @@ -65,7 +65,7 @@ open class CallExpression : /** The list of arguments of this call expression, backed by a list of [Edge] objects. */ @Relationship(value = "ARGUMENTS", direction = Relationship.Direction.OUTGOING) - var argumentEdges = astEdgesOf() + var argumentEdges = astEdgesOf(label = "ARGUMENTS") /** * The list of arguments as a simple list. This is a delegated property delegated to @@ -85,7 +85,8 @@ open class CallExpression : * is intentionally left empty. It is not filled by the [SymbolResolver]. */ @Relationship(value = "CALLEE", direction = Relationship.Direction.OUTGOING) - private var calleeEdge = astEdgeOf(ProblemExpression("could not parse callee")) + private var calleeEdge = + astEdgeOf(ProblemExpression("could not parse callee"), label = "CALLEE") var callee by unwrapping(CallExpression::calleeEdge) @@ -120,7 +121,7 @@ open class CallExpression : /** Adds the specified [expression] with an optional [name] to this call. */ fun addArgument(expression: Expression, name: String? = null) { - val edge = AstEdge(this, expression) + val edge = AstEdge(this, expression, label = "ARGUMENTS") edge.name = name argumentEdges.add(edge) @@ -185,7 +186,7 @@ open class CallExpression : ) { if (templateParam is Expression || templateParam is Type) { if (templateArgumentEdges == null) { - templateArgumentEdges = TemplateArguments(this) + templateArgumentEdges = TemplateArguments(this, label = "TEMPLATE_ARGUMENTS") } templateArgumentEdges?.add(templateParam) { instantiation = templateInitialization } @@ -198,7 +199,7 @@ open class CallExpression : orderedInitializationSignature: List ) { if (templateArgumentEdges == null) { - templateArgumentEdges = TemplateArguments(this) + templateArgumentEdges = TemplateArguments(this, label = "TEMPLATE_ARGUMENTS") } for (edge in templateArgumentEdges ?: listOf()) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CastExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CastExpression.kt index 541cf38a255..19f9f54eceb 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CastExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CastExpression.kt @@ -48,6 +48,7 @@ class CastExpression : Expression(), ArgumentHolder, HasType.TypeObserver { var expressionEdge = astEdgeOf( of = ProblemExpression("could not parse inner expression"), + label = "EXPRESSION", onChanged = ::exchangeTypeObserver ) var expression by unwrapping(CastExpression::expressionEdge) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CollectionComprehension.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CollectionComprehension.kt index e51e306f3d9..d75a716c2f1 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CollectionComprehension.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CollectionComprehension.kt @@ -46,7 +46,8 @@ import org.neo4j.ogm.annotation.Relationship class CollectionComprehension : Expression(), ArgumentHolder { @Relationship("COMPREHENSION_EXPRESSIONS") - var comprehensionExpressionEdges = astEdgesOf() + var comprehensionExpressionEdges = + astEdgesOf(label = "COMPREHENSION_EXPRESSIONS") /** * This field contains one or multiple [ComprehensionExpression]s. * @@ -60,7 +61,8 @@ class CollectionComprehension : Expression(), ArgumentHolder { @Relationship("STATEMENT") var statementEdge = astEdgeOf( - ProblemExpression("No statement provided but is required in ${this::class}") + ProblemExpression("No statement provided but is required in ${this::class}"), + label = "STATEMENT", ) /** * This field contains the statement which is applied to each element of the input for which the diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ComprehensionExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ComprehensionExpression.kt index aeb24c0bf20..4b8f87158d2 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ComprehensionExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ComprehensionExpression.kt @@ -41,6 +41,7 @@ class ComprehensionExpression : Expression(), ArgumentHolder { var variableEdge = astEdgeOf( of = ProblemExpression("Missing variableEdge in ${this::class}"), + label = "VARIABLE", onChanged = { _, new -> val end = new?.end if (end is Reference) { @@ -57,12 +58,15 @@ class ComprehensionExpression : Expression(), ArgumentHolder { @Relationship("ITERABLE") var iterableEdge = - astEdgeOf(ProblemExpression("Missing iterable in ${this::class}")) + astEdgeOf( + ProblemExpression("Missing iterable in ${this::class}"), + label = "ITERABLE" + ) /** This field contains the iteration subject of the loop. */ var iterable by unwrapping(ComprehensionExpression::iterableEdge) - @Relationship("PREDICATE") var predicateEdge = astOptionalEdgeOf() + @Relationship("PREDICATE") var predicateEdge = astOptionalEdgeOf(label = "PREDICATE") /** * This field contains the predicate which has to hold to evaluate `statement(variable)` and diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConditionalExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConditionalExpression.kt index 977c85d8ed3..bc8b0a85e8d 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConditionalExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConditionalExpression.kt @@ -43,12 +43,16 @@ import org.neo4j.ogm.annotation.Relationship class ConditionalExpression : Expression(), ArgumentHolder, BranchingNode, HasType.TypeObserver { @Relationship("CONDITION") var conditionEdge = - astEdgeOf(ProblemExpression("could not parse condition expression")) + astEdgeOf( + ProblemExpression("could not parse condition expression"), + label = "CONDITION" + ) var condition by unwrapping(ConditionalExpression::conditionEdge) @Relationship("THEN_EXPRESSION") var thenExpressionEdge = astOptionalEdgeOf( + label = "THEN_EXPRESSION", onChanged = { old, new -> old?.end?.unregisterTypeObserver(this) new?.end?.registerTypeObserver(this) @@ -59,6 +63,7 @@ class ConditionalExpression : Expression(), ArgumentHolder, BranchingNode, HasTy @Relationship("ELSE_EXPRESSION") var elseExpressionEdge = astOptionalEdgeOf( + label = "ELSE_EXPRESSION", onChanged = { old, new -> old?.end?.unregisterTypeObserver(this) new?.end?.registerTypeObserver(this) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConstructExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConstructExpression.kt index c6c81d51ae8..c9f3d3535aa 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConstructExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConstructExpression.kt @@ -65,7 +65,8 @@ class ConstructExpression : CallExpression() { } } - @Relationship("ANONYMOUS_CLASS") var anonymousClassEdge = astOptionalEdgeOf() + @Relationship("ANONYMOUS_CLASS") + var anonymousClassEdge = astOptionalEdgeOf(label = "ANONYMOUS_CLASS") var anonymousClass by unwrapping(ConstructExpression::anonymousClassEdge) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/DeleteExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/DeleteExpression.kt index dfa39b08f2f..2e969393950 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/DeleteExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/DeleteExpression.kt @@ -31,7 +31,7 @@ import java.util.Objects import org.neo4j.ogm.annotation.Relationship class DeleteExpression : Expression() { - @Relationship("OPERANDS") var operandEdges = astEdgesOf() + @Relationship("OPERANDS") var operandEdges = astEdgesOf(label = "OPERANDS") var operands by unwrapping(DeleteExpression::operandEdges) override fun equals(other: Any?): Boolean { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ExpressionList.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ExpressionList.kt index f349cc7a332..be7e87711d5 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ExpressionList.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ExpressionList.kt @@ -34,7 +34,7 @@ import org.neo4j.ogm.annotation.Relationship class ExpressionList : Expression() { @Relationship(value = "SUBEXPR", direction = Relationship.Direction.OUTGOING) - var expressionEdges = astEdgesOf() + var expressionEdges = astEdgesOf(label = "SUBEXPR") var expressions by unwrapping(ExpressionList::expressionEdges) override fun equals(other: Any?): Boolean { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt index 62fa90ca8ea..bf04043718a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt @@ -50,6 +50,7 @@ class InitializerListExpression : Expression(), ArgumentHolder, HasType.TypeObse astEdgesOf( onAdd = { it.end.registerTypeObserver(this) }, onRemove = { it.end.unregisterTypeObserver(this) }, + label = "INITIALIZERS", ) /** Virtual property to access [initializerEdges] without property edges. */ diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/KeyValueExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/KeyValueExpression.kt index d6b2eaff816..04325a7d815 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/KeyValueExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/KeyValueExpression.kt @@ -40,14 +40,16 @@ import org.neo4j.ogm.annotation.Relationship */ class KeyValueExpression : Expression(), ArgumentHolder { - @Relationship("KEY") var keyEdge = astEdgeOf(ProblemExpression("missing key")) + @Relationship("KEY") + var keyEdge = astEdgeOf(ProblemExpression("missing key"), label = "KEY") /** * The key of this pair. It is usually a literal, but some languages even allow references to * variables as a key. */ var key by unwrapping(KeyValueExpression::keyEdge) - @Relationship("VALUE") var valueEdge = astEdgeOf(ProblemExpression("missing value")) + @Relationship("VALUE") + var valueEdge = astEdgeOf(ProblemExpression("missing value"), label = "VALUE") /** The value of this pair. It can be any expression */ var value by unwrapping(KeyValueExpression::valueEdge) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/LambdaExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/LambdaExpression.kt index 2b8823be979..5827f3ebe1b 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/LambdaExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/LambdaExpression.kt @@ -51,7 +51,11 @@ class LambdaExpression : Expression(), HasType.TypeObserver { var areVariablesMutable: Boolean = true @Relationship("FUNCTION") - var functionEdge = astOptionalEdgeOf(onChanged = ::exchangeTypeObserver) + var functionEdge = + astOptionalEdgeOf( + label = "FUNCTION", + onChanged = ::exchangeTypeObserver + ) var function by unwrapping(LambdaExpression::functionEdge) override fun typeChanged(newType: Type, src: HasType) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.kt index 21c0e2cb85c..bee94e37e58 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.kt @@ -48,6 +48,7 @@ class MemberExpression : Reference(), HasOverloadedOperation, ArgumentHolder, Ha var baseEdge = astEdgeOf( ProblemExpression("could not parse base expression"), + label = "BASE", onChanged = { old, new -> exchangeTypeObserver(old, new) updateName() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/NewArrayExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/NewArrayExpression.kt index d0bfa683f22..4f4ad5d0fd7 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/NewArrayExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/NewArrayExpression.kt @@ -39,7 +39,8 @@ import org.neo4j.ogm.annotation.Relationship */ // TODO Merge and/or refactor with new Expression class NewArrayExpression : Expression() { - @Relationship("INITIALIZER") var initializerEdge = astOptionalEdgeOf() + @Relationship("INITIALIZER") + var initializerEdge = astOptionalEdgeOf(label = "INITIALIZER") /** * The initializer of the expression, if present. Many languages, such as Java, either specify @@ -53,7 +54,7 @@ class NewArrayExpression : Expression() { * dimensions. In the graph, this will NOT be done. */ @Relationship(value = "DIMENSIONS", direction = Relationship.Direction.OUTGOING) - var dimensionEdges = astEdgesOf() + var dimensionEdges = astEdgesOf(label = "DIMENSIONS") /** Virtual property to access [dimensionEdges] without property edges. */ var dimensions by unwrapping(NewArrayExpression::dimensionEdges) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/NewExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/NewExpression.kt index 807dbcdf296..9132992c405 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/NewExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/NewExpression.kt @@ -36,7 +36,8 @@ import org.neo4j.ogm.annotation.Relationship /** Represents the creation of a new object through the `new` keyword. */ class NewExpression : Expression(), HasInitializer { - @Relationship("INITIALIZER") var initializerEdge = astOptionalEdgeOf() + @Relationship("INITIALIZER") + var initializerEdge = astOptionalEdgeOf(label = "INITIALIZER") /** The initializer expression. */ override var initializer by unwrapping(NewExpression::initializerEdge) @@ -46,7 +47,7 @@ class NewExpression : Expression(), HasInitializer { * ConstructExpression is created */ @Relationship(value = "TEMPLATE_PARAMETERS", direction = Relationship.Direction.OUTGOING) - var templateParameterEdges = astEdgesOf() + var templateParameterEdges = astEdgesOf(label = "TEMPLATE_PARAMETERS") var templateParameters by unwrapping(NewExpression::templateParameterEdges) override fun toString(): String { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/RangeExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/RangeExpression.kt index 0140852024c..180bdc84801 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/RangeExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/RangeExpression.kt @@ -55,15 +55,15 @@ import org.neo4j.ogm.annotation.Relationship * Individual meaning of the range indices might differ per language. */ class RangeExpression : Expression() { - @Relationship("FLOOR") var floorEdge = astOptionalEdgeOf() + @Relationship("FLOOR") var floorEdge = astOptionalEdgeOf(label = "FLOOR") /** The lower bound ("floor") of the range. This index is usually *inclusive*. */ var floor by unwrapping(RangeExpression::floorEdge) - @Relationship("CEILING") var ceilingEdge = astOptionalEdgeOf() + @Relationship("CEILING") var ceilingEdge = astOptionalEdgeOf(label = "CEILING") /** The upper bound ("ceiling") of the range. This index is usually *exclusive*. */ var ceiling by unwrapping(RangeExpression::ceilingEdge) - @Relationship("THIRD") var thirdEdge = astOptionalEdgeOf() + @Relationship("THIRD") var thirdEdge = astOptionalEdgeOf(label = "THIRD") /** * Some languages offer a third value. The meaning depends completely on the language. For * example, Python allows specifying a step, while Go allows to control the underlying array's diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/SubscriptExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/SubscriptExpression.kt index 8254241a749..b4b3a70d552 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/SubscriptExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/SubscriptExpression.kt @@ -43,6 +43,7 @@ class SubscriptExpression : Expression(), HasBase, HasType.TypeObserver, Argumen var arrayExpressionEdge = astEdgeOf( of = ProblemExpression("could not parse array expression"), + label = "ARRAY_EXPRESSION", onChanged = ::exchangeTypeObserver ) /** The array on which the access is happening. This is most likely a [Reference]. */ @@ -50,7 +51,10 @@ class SubscriptExpression : Expression(), HasBase, HasType.TypeObserver, Argumen @Relationship("SUBSCRIPT_EXPRESSION") var subscriptExpressionEdge = - astEdgeOf(ProblemExpression("could not parse index expression")) + astEdgeOf( + ProblemExpression("could not parse index expression"), + label = "SUBSCRIPT_EXPRESSION" + ) /** * The expression which represents the "subscription" or index on which the array is accessed. * This can for example be a reference to another variable ([Reference]), a [Literal] or a diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/UnaryOperator.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/UnaryOperator.kt index 3bb1931a570..cfab59fc844 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/UnaryOperator.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/UnaryOperator.kt @@ -42,6 +42,7 @@ class UnaryOperator : Expression(), HasOverloadedOperation, ArgumentHolder, HasT var inputEdge = astEdgeOf( of = ProblemExpression("could not parse input"), + label = "INPUT", onChanged = { old, new -> exchangeTypeObserver(old, new) changeExpressionAccess() diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt index 7cb110f6231..4171be655ba 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt @@ -27,9 +27,13 @@ package de.fraunhofer.aisec.cpg.v2 import de.fraunhofer.aisec.cpg.TranslationResult import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.edges.Edge +import de.fraunhofer.aisec.cpg.graph.edges.allEdges +import de.fraunhofer.aisec.cpg.graph.edges.edges import de.fraunhofer.aisec.cpg.graph.nodes import kotlin.collections.joinToString import kotlin.reflect.KClass +import kotlin.uuid.Uuid import org.neo4j.driver.GraphDatabase import org.neo4j.driver.TransactionContext @@ -50,19 +54,42 @@ fun TranslationResult.persist() { for (node in nodes) { node.persist() } + + val edges = this@persist.allEdges>() + for (edge in edges) { + edge.persist() + } } } } context(TransactionContext) fun Node.persist() { + if (this.graphId == null) { + this.graphId = Uuid.random() + } + + val result = + this@TransactionContext.run( + "MERGE (n:${this.labels.joinToString("&")} { name: \$name, id: \$id } ) RETURN n.id", + mapOf( + "name" to this.name.localName, + "id" to this.graphId.toString(), + ) + ) + println("Created node with id $graphId") +} + +context(TransactionContext) +fun Edge<*>.persist() { val result = this@TransactionContext.run( - "MERGE (n:${this.labels.joinToString("&")} { name: \$name } ) RETURN elementId(n) AS id", - mapOf("name" to this.name.localName) + "MATCH (start { id: \$startId }), (end { id: \$endId } ) MERGE (start)-[r:${label}]->(end)", + mapOf( + "startId" to this.start.graphId.toString(), + "endId" to this.end.graphId.toString() + ) ) - val id = result.single()["id"] - println("Created node with id $id") } val Node.labels: Set diff --git a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg/v2/TestPersistence.kt b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg/v2/TestPersistence.kt index 2f9f7daa673..9ec02f48961 100644 --- a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg/v2/TestPersistence.kt +++ b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg/v2/TestPersistence.kt @@ -34,6 +34,8 @@ class TestPersistence { @Test fun testPersist() { val result = createTranslationResult() + + neo4jSession.executeWrite { tx -> tx.run("MATCH (n) DETACH DELETE n").consume() } result.second.persist() } } From 6101f572227ee8929c9e038fb983d9bf099f8ae8 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Tue, 10 Dec 2024 22:12:13 +0100 Subject: [PATCH 03/25] Setting labels of all edges --- .../main/kotlin/de/fraunhofer/aisec/cpg/graph/Assignment.kt | 4 +++- .../main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt | 2 +- .../kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt | 4 ++-- .../fraunhofer/aisec/cpg/graph/edges/flows/EvaluationOrder.kt | 2 ++ .../de/fraunhofer/aisec/cpg/graph/edges/flows/Invoke.kt | 2 ++ .../kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Usage.kt | 2 ++ 6 files changed, 12 insertions(+), 4 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Assignment.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Assignment.kt index d9a32dbe22c..6cc946d0d16 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Assignment.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Assignment.kt @@ -45,4 +45,6 @@ class Assignment( /** The holder of this assignment */ @JsonIgnore val holder: AssignmentHolder -) : Edge(value, target as Node) +) : Edge(value, target as Node) { + override var label: String = "ASSIGMENT" +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt index 67428529070..8b3d5dea424 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt @@ -71,7 +71,7 @@ abstract class Edge : Persistable, Cloneable { end = edge.end } - @Transient open var label: String = "EDGE" + abstract var label: String /** * The index of this node, if it is stored in an diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt index cd2d2f9fafd..a83d7ed01db 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt @@ -33,10 +33,10 @@ import org.neo4j.ogm.annotation.* /** This property edge describes a parent/child relationship in the Abstract Syntax Tree (AST). */ @RelationshipEntity -open class AstEdge(start: Node, end: T, label: String = "AST") : Edge(start, end) { +open class AstEdge(start: Node, end: T, override var label: String = "AST") : + Edge(start, end) { init { end.astParent = start - this.label = label } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/EvaluationOrder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/EvaluationOrder.kt index af99e4dcaf7..e6af5e8d818 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/EvaluationOrder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/EvaluationOrder.kt @@ -68,6 +68,8 @@ class EvaluationOrder( result = 31 * result + branch.hashCode() return result } + + override var label: String = "EOG" } /** diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Invoke.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Invoke.kt index 261b4957e4f..e6a94996835 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Invoke.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Invoke.kt @@ -43,6 +43,8 @@ class Invoke( */ var dynamicInvoke: Boolean = false, ) : Edge(start, end) { + override var label: String = "INVOKES" + override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is Invoke) return false diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Usage.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Usage.kt index f5c7a863d75..b4a41a6821a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Usage.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Usage.kt @@ -52,6 +52,8 @@ class Usage( result = 31 * result + access.hashCode() return result } + + override var label: String = "USAGE" } /** A container for [Usage] edges. [NodeType] is necessary because of the Neo4J OGM. */ From 45ce1566dc6ddd54392cb8a4cda05d156b66a14c Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Tue, 10 Dec 2024 22:54:02 +0100 Subject: [PATCH 04/25] Add node properties --- .../de/fraunhofer/aisec/cpg/graph/Node.kt | 6 +- .../graph/declarations/FieldDeclaration.kt | 2 +- .../de/fraunhofer/aisec/cpg/v2/Persistence.kt | 100 ++++++++++++++---- 3 files changed, 86 insertions(+), 22 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt index d4715d5ac05..9f074aaf7c1 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt @@ -253,10 +253,10 @@ abstract class Node : var isImplicit = false /** Required field for object graph mapping. It contains the node id. */ - @Id @GeneratedValue var id: Long? = null + @Id @GeneratedValue var legacyId: Long? = null - /** Will replace [id] */ - var graphId: Uuid? = null + /** Will replace [legacyId] */ + var id: Uuid? = null /** Index of the argument if this node is used in a function call or parameter list. */ var argumentIndex = 0 diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FieldDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FieldDeclaration.kt index 6986ec21e20..bcfbd1a9ceb 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FieldDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FieldDeclaration.kt @@ -36,7 +36,7 @@ import org.neo4j.ogm.annotation.Relationship */ class FieldDeclaration : VariableDeclaration() { /** Specifies, whether this field declaration is also a definition, i.e. has an initializer. */ - private var isDefinition = false + var isDefinition = false /** If this is only a declaration, this provides a link to the definition of the field. */ @Relationship(value = "DEFINES") diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt index 4171be655ba..8a410fe4bcc 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt @@ -26,13 +26,19 @@ package de.fraunhofer.aisec.cpg.v2 import de.fraunhofer.aisec.cpg.TranslationResult +import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.edges.Edge import de.fraunhofer.aisec.cpg.graph.edges.allEdges -import de.fraunhofer.aisec.cpg.graph.edges.edges import de.fraunhofer.aisec.cpg.graph.nodes +import de.fraunhofer.aisec.cpg.helpers.neo4j.NameConverter +import de.fraunhofer.aisec.cpg.helpers.neo4j.SimpleNameConverter import kotlin.collections.joinToString import kotlin.reflect.KClass +import kotlin.reflect.KProperty1 +import kotlin.reflect.full.createType +import kotlin.reflect.full.memberProperties +import kotlin.reflect.full.withNullability import kotlin.uuid.Uuid import org.neo4j.driver.GraphDatabase import org.neo4j.driver.TransactionContext @@ -47,6 +53,9 @@ val neo4jSession by lazy { val labelCache: MutableMap, Set> = mutableMapOf() +val schemaPropertiesCache: MutableMap, Map>> = + mutableMapOf() + fun TranslationResult.persist() { neo4jSession.executeWrite { tx -> with(tx) { @@ -65,31 +74,59 @@ fun TranslationResult.persist() { context(TransactionContext) fun Node.persist() { - if (this.graphId == null) { - this.graphId = Uuid.random() + if (this.id == null) { + this.id = Uuid.random() } - val result = - this@TransactionContext.run( - "MERGE (n:${this.labels.joinToString("&")} { name: \$name, id: \$id } ) RETURN n.id", - mapOf( - "name" to this.name.localName, - "id" to this.graphId.toString(), - ) + val properties = this.properties() + + this@TransactionContext.run( + "MERGE (n:${this.labels.joinToString("&")} ${properties.writePlaceHolder()} ) RETURN n.id", + this.properties() ) - println("Created node with id $graphId") + .consume() + println("Created node with id $id") +} + +fun Map.writePlaceHolder(): String { + return this.map { "${it.key}: \$${it.key}" }.joinToString(", ", prefix = "{ ", postfix = " }") +} + +/** + * Returns the node's properties. This DOES NOT include relationships, but only properties directly + * attached to the node. + */ +fun Node.properties(): Map { + val properties = mutableMapOf() + for (entry in schemaProperties) { + val value = entry.value.call(this) + + if (value == null) { + continue + } + + // TODO: generalize conversions + if (value is Name && entry.key == "name") { + properties += NameConverter().toGraphProperties(value) + } else if (value is Name) { + properties.put(entry.key, SimpleNameConverter().toGraphProperty(value)) + } else if (value is Uuid) { + properties.put(entry.key, value.toString()) + } else { + properties.put(entry.key, value) + } + } + + return properties } context(TransactionContext) fun Edge<*>.persist() { - val result = - this@TransactionContext.run( - "MATCH (start { id: \$startId }), (end { id: \$endId } ) MERGE (start)-[r:${label}]->(end)", - mapOf( - "startId" to this.start.graphId.toString(), - "endId" to this.end.graphId.toString() - ) + this@TransactionContext.run( + "MATCH (start { id: \$startId }), (end { id: \$endId } ) MERGE (start)-[r:${label} { }]->(end)", + mapOf("startId" to this.start.id.toString(), "endId" to this.end.id.toString()) ) + .consume() } val Node.labels: Set @@ -99,3 +136,30 @@ val Node.labels: Set // Check, if we already computed the labels for this node's class return labelCache.computeIfAbsent(klazz) { setOf("Node", klazz.simpleName!!) } } + +val primitiveTypes = + setOf( + String::class.createType(), + Int::class.createType(), + Long::class.createType(), + Boolean::class.createType(), + Name::class.createType(), + Uuid::class.createType(), + ) + +val Node.schemaProperties: Map> + get() { + val klazz = this::class + + // Check, if we already computed the labels for this node's class + return schemaPropertiesCache.computeIfAbsent(klazz) { + val schema = mutableMapOf>() + val properties = it.memberProperties + for (property in properties) { + if (property.returnType.withNullability(false) in primitiveTypes) { + schema.put(property.name, property) + } + } + schema + } + } From bb1668fa5bb1c33bf2645ff3eae170eecc8e943a Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Tue, 10 Dec 2024 23:44:13 +0100 Subject: [PATCH 05/25] Batching everything --- .../de/fraunhofer/aisec/cpg/graph/Node.kt | 2 +- .../de/fraunhofer/aisec/cpg/v2/Persistence.kt | 99 ++++++++++++++----- .../aisec/cpg/v2/TestPersistence.kt | 25 +++++ 3 files changed, 99 insertions(+), 27 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt index 9f074aaf7c1..948be849757 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt @@ -256,7 +256,7 @@ abstract class Node : @Id @GeneratedValue var legacyId: Long? = null /** Will replace [legacyId] */ - var id: Uuid? = null + var id: Uuid = Uuid.random() /** Index of the argument if this node is used in a function call or parameter list. */ var argumentIndex = 0 diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt index 8a410fe4bcc..c583a65c6ce 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt @@ -42,7 +42,12 @@ import kotlin.reflect.full.withNullability import kotlin.uuid.Uuid import org.neo4j.driver.GraphDatabase import org.neo4j.driver.TransactionContext +import org.slf4j.LoggerFactory +/** + * docker run \ --name neo4j-apoc \ -p 7474:7474 -p 7687:7687 \ -d \ -e NEO4J_AUTH=neo4j/password \ + * -e NEO4JLABS_PLUGINS='["apoc"]' \ neo4j:5 + */ val dbUri = "neo4j://localhost" val dbUser = "neo4j" val dbPassword = "password" @@ -56,40 +61,76 @@ val labelCache: MutableMap, Set> = mutableMapOf() val schemaPropertiesCache: MutableMap, Map>> = mutableMapOf() +val log = LoggerFactory.getLogger("Persistence") + fun TranslationResult.persist() { - neo4jSession.executeWrite { tx -> - with(tx) { - val nodes = this@persist.nodes - for (node in nodes) { - node.persist() - } + val nodes = this@persist.nodes + val edges = this@persist.allEdges>() + + log.info("Persisting {} nodes", nodes.size) + nodes.persist() - val edges = this@persist.allEdges>() - for (edge in edges) { - edge.persist() + log.info("Persisting {} edges", edges.size) + edges.persist() +} + +private fun List.persist() { + val groups = groupBy { it::class } + groups.forEach { + it.value.chunked(10000).forEach { chunk -> + log.info("Processing ${chunk.size} nodes of type ${it.key}") + + val params = mapOf("props" to chunk.map { it.properties() }) + val start = System.currentTimeMillis() + neo4jSession.executeWrite { tx -> + tx.run( + "UNWIND \$props AS map CREATE (n:${it.key.labels.joinToString("&")}) SET n=map", + params + ) + .consume() } + log.info( + "Time Taken to process and save ${chunk.size} records to Neo4j Batch Insert took ${System.currentTimeMillis() - start} ms" + ) } } } -context(TransactionContext) -fun Node.persist() { - if (this.id == null) { - this.id = Uuid.random() +private fun Collection>.persist() { + val groups = groupBy { it.label } + groups.forEach { + it.value.chunked(10000).forEach { chunk -> + log.info("Processing ${chunk.size} edges of type ${it.key}") + + val params = + mapOf( + "props" to + chunk.map { + mapOf( + "startId" to it.start.id.toString(), + "endId" to it.end.id.toString(), + ) + } + ) + val start = System.currentTimeMillis() + neo4jSession.executeWrite { tx -> + tx.run( + """ + UNWIND ${'$'}props AS map + MATCH (s {id: map.startId}) + MATCH (e {id: map.endId}) + CREATE (s)-[r:${it.key} {}]->(e) + """ + .trimIndent(), + params + ) + .consume() + } + log.info( + "Time Taken to process and save ${chunk.size} records to Neo4j Batch Insert took ${System.currentTimeMillis() - start} ms" + ) + } } - - val properties = this.properties() - - this@TransactionContext.run( - "MERGE (n:${this.labels.joinToString("&")} ${properties.writePlaceHolder()} ) RETURN n.id", - this.properties() - ) - .consume() - println("Created node with id $id") -} - -fun Map.writePlaceHolder(): String { - return this.map { "${it.key}: \$${it.key}" }.joinToString(", ", prefix = "{ ", postfix = " }") } /** @@ -137,6 +178,12 @@ val Node.labels: Set return labelCache.computeIfAbsent(klazz) { setOf("Node", klazz.simpleName!!) } } +val KClass.labels: Set + get() { + // Check, if we already computed the labels for this node's class + return labelCache.computeIfAbsent(this) { setOf("Node", this.simpleName!!) } + } + val primitiveTypes = setOf( String::class.createType(), diff --git a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg/v2/TestPersistence.kt b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg/v2/TestPersistence.kt index 9ec02f48961..b1f48c0aeec 100644 --- a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg/v2/TestPersistence.kt +++ b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg/v2/TestPersistence.kt @@ -25,7 +25,10 @@ */ package de.fraunhofer.aisec.cpg.v2 +import de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage +import de.fraunhofer.aisec.cpg.test.analyze import de.fraunhofer.aisec.cpg_vis_neo4j.createTranslationResult +import java.nio.file.Path import org.junit.jupiter.api.Tag import org.junit.jupiter.api.Test @@ -38,4 +41,26 @@ class TestPersistence { neo4jSession.executeWrite { tx -> tx.run("MATCH (n) DETACH DELETE n").consume() } result.second.persist() } + + @Test + fun testPersistGlance() { + val topLevel = + Path.of("/Users/chr55316/Repositories/openstack-checker/targets/projects/glance") + val result = + analyze( + listOf( + topLevel.resolve("glance").toFile(), + ), + topLevel, + true + ) { + it.registerLanguage() + it.exclusionPatterns("tests") + it.useParallelFrontends(false) + it.failOnError(false) + } + + neo4jSession.executeWrite { tx -> tx.run("MATCH (n) DETACH DELETE n").consume() } + result.persist() + } } From 4f689f5780fc6af53610b112a3ff91fefc8a3634 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Tue, 10 Dec 2024 23:54:49 +0100 Subject: [PATCH 06/25] Adding index --- .../main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt | 8 ++++++-- .../kotlin/de/fraunhofer/aisec/cpg/v2/TestPersistence.kt | 3 +++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt index c583a65c6ce..376ea5ea969 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt @@ -67,6 +67,10 @@ fun TranslationResult.persist() { val nodes = this@persist.nodes val edges = this@persist.allEdges>() + neo4jSession.executeWrite { tx -> + tx.run("CREATE INDEX IF NOT EXISTS FOR (n:Node) ON (n.id)").consume() + } + log.info("Persisting {} nodes", nodes.size) nodes.persist() @@ -117,8 +121,8 @@ private fun Collection>.persist() { tx.run( """ UNWIND ${'$'}props AS map - MATCH (s {id: map.startId}) - MATCH (e {id: map.endId}) + MATCH (s:Node {id: map.startId}) + MATCH (e:Node {id: map.endId}) CREATE (s)-[r:${it.key} {}]->(e) """ .trimIndent(), diff --git a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg/v2/TestPersistence.kt b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg/v2/TestPersistence.kt index b1f48c0aeec..7685a0df950 100644 --- a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg/v2/TestPersistence.kt +++ b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg/v2/TestPersistence.kt @@ -26,6 +26,7 @@ package de.fraunhofer.aisec.cpg.v2 import de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage +import de.fraunhofer.aisec.cpg.helpers.Benchmark import de.fraunhofer.aisec.cpg.test.analyze import de.fraunhofer.aisec.cpg_vis_neo4j.createTranslationResult import java.nio.file.Path @@ -60,7 +61,9 @@ class TestPersistence { it.failOnError(false) } + val bench = Benchmark(this.javaClass, "Persist") neo4jSession.executeWrite { tx -> tx.run("MATCH (n) DETACH DELETE n").consume() } result.persist() + bench.stop() } } From 9f95d9f23567d29d248646feb4b5e667976e322f Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 11 Dec 2024 00:05:16 +0100 Subject: [PATCH 07/25] Fixed --- .../de/fraunhofer/aisec/cpg/v2/Persistence.kt | 8 +++--- .../aisec/cpg/v2/TestPersistence.kt | 28 ------------------- 2 files changed, 4 insertions(+), 32 deletions(-) diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt index 376ea5ea969..46929daa33b 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt @@ -67,10 +67,6 @@ fun TranslationResult.persist() { val nodes = this@persist.nodes val edges = this@persist.allEdges>() - neo4jSession.executeWrite { tx -> - tx.run("CREATE INDEX IF NOT EXISTS FOR (n:Node) ON (n.id)").consume() - } - log.info("Persisting {} nodes", nodes.size) nodes.persist() @@ -101,6 +97,10 @@ private fun List.persist() { } private fun Collection>.persist() { + neo4jSession.executeWrite { tx -> + tx.run("CREATE INDEX IF NOT EXISTS FOR (n:Node) ON (n.id)").consume() + } + val groups = groupBy { it.label } groups.forEach { it.value.chunked(10000).forEach { chunk -> diff --git a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg/v2/TestPersistence.kt b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg/v2/TestPersistence.kt index 7685a0df950..9ec02f48961 100644 --- a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg/v2/TestPersistence.kt +++ b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg/v2/TestPersistence.kt @@ -25,11 +25,7 @@ */ package de.fraunhofer.aisec.cpg.v2 -import de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage -import de.fraunhofer.aisec.cpg.helpers.Benchmark -import de.fraunhofer.aisec.cpg.test.analyze import de.fraunhofer.aisec.cpg_vis_neo4j.createTranslationResult -import java.nio.file.Path import org.junit.jupiter.api.Tag import org.junit.jupiter.api.Test @@ -42,28 +38,4 @@ class TestPersistence { neo4jSession.executeWrite { tx -> tx.run("MATCH (n) DETACH DELETE n").consume() } result.second.persist() } - - @Test - fun testPersistGlance() { - val topLevel = - Path.of("/Users/chr55316/Repositories/openstack-checker/targets/projects/glance") - val result = - analyze( - listOf( - topLevel.resolve("glance").toFile(), - ), - topLevel, - true - ) { - it.registerLanguage() - it.exclusionPatterns("tests") - it.useParallelFrontends(false) - it.failOnError(false) - } - - val bench = Benchmark(this.javaClass, "Persist") - neo4jSession.executeWrite { tx -> tx.run("MATCH (n) DETACH DELETE n").consume() } - result.persist() - bench.stop() - } } From 4034add1f34e605d5725751f8d04d7b39d140dac Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 11 Dec 2024 09:42:53 +0100 Subject: [PATCH 08/25] Persisting edge properties --- .../de/fraunhofer/aisec/cpg/v2/Persistence.kt | 177 +++++++++--------- 1 file changed, 92 insertions(+), 85 deletions(-) diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt index 46929daa33b..26664a7533d 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt @@ -28,12 +28,17 @@ package de.fraunhofer.aisec.cpg.v2 import de.fraunhofer.aisec.cpg.TranslationResult import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.Persistable import de.fraunhofer.aisec.cpg.graph.edges.Edge import de.fraunhofer.aisec.cpg.graph.edges.allEdges +import de.fraunhofer.aisec.cpg.graph.edges.flows.DependenceType +import de.fraunhofer.aisec.cpg.graph.edges.flows.Granularity import de.fraunhofer.aisec.cpg.graph.nodes +import de.fraunhofer.aisec.cpg.helpers.Benchmark +import de.fraunhofer.aisec.cpg.helpers.neo4j.DataflowGranularityConverter import de.fraunhofer.aisec.cpg.helpers.neo4j.NameConverter import de.fraunhofer.aisec.cpg.helpers.neo4j.SimpleNameConverter -import kotlin.collections.joinToString +import kotlin.collections.plusAssign import kotlin.reflect.KClass import kotlin.reflect.KProperty1 import kotlin.reflect.full.createType @@ -41,7 +46,7 @@ import kotlin.reflect.full.memberProperties import kotlin.reflect.full.withNullability import kotlin.uuid.Uuid import org.neo4j.driver.GraphDatabase -import org.neo4j.driver.TransactionContext +import org.neo4j.ogm.typeconversion.CompositeAttributeConverter import org.slf4j.LoggerFactory /** @@ -53,134 +58,136 @@ val dbUser = "neo4j" val dbPassword = "password" val neo4jSession by lazy { - GraphDatabase.driver(dbUri, org.neo4j.driver.AuthTokens.basic(dbUser, dbPassword)).session() + val driver = GraphDatabase.driver(dbUri, org.neo4j.driver.AuthTokens.basic(dbUser, dbPassword)) + driver.session() } val labelCache: MutableMap, Set> = mutableMapOf() -val schemaPropertiesCache: MutableMap, Map>> = +val schemaPropertiesCache: + MutableMap, Map>> = mutableMapOf() val log = LoggerFactory.getLogger("Persistence") +val edgeChunkSize = 10000 +val nodeChunkSize = 10000 + fun TranslationResult.persist() { val nodes = this@persist.nodes val edges = this@persist.allEdges>() + val b = Benchmark(Persistable::class.java, "Persisting translation result") + log.info("Persisting {} nodes", nodes.size) nodes.persist() log.info("Persisting {} edges", edges.size) edges.persist() + + b.stop() } private fun List.persist() { - val groups = groupBy { it::class } - groups.forEach { - it.value.chunked(10000).forEach { chunk -> - log.info("Processing ${chunk.size} nodes of type ${it.key}") - - val params = mapOf("props" to chunk.map { it.properties() }) - val start = System.currentTimeMillis() - neo4jSession.executeWrite { tx -> - tx.run( - "UNWIND \$props AS map CREATE (n:${it.key.labels.joinToString("&")}) SET n=map", - params - ) - .consume() - } - log.info( - "Time Taken to process and save ${chunk.size} records to Neo4j Batch Insert took ${System.currentTimeMillis() - start} ms" - ) + this.chunked(nodeChunkSize).map { chunk -> + val b = Benchmark(Persistable::class.java, "Persisting chunk of ${chunk.size} nodes") + val params = + mapOf("props" to chunk.map { mapOf("labels" to it::class.labels) + it.properties() }) + neo4jSession.executeWrite { tx -> + tx.run( + """ + UNWIND ${"$"}props AS map + WITH map, apoc.map.removeKeys(map, ['labels']) AS properties + CALL apoc.create.node(map.labels, properties) YIELD node + RETURN node + """, + params + ) + .consume() } + b.stop() } } private fun Collection>.persist() { + // Create an index for the "id" field of node, because we are "MATCH"ing on it in the edge + // creation. We need to wait for this to be finished neo4jSession.executeWrite { tx -> tx.run("CREATE INDEX IF NOT EXISTS FOR (n:Node) ON (n.id)").consume() } - val groups = groupBy { it.label } - groups.forEach { - it.value.chunked(10000).forEach { chunk -> - log.info("Processing ${chunk.size} edges of type ${it.key}") - - val params = - mapOf( - "props" to - chunk.map { - mapOf( - "startId" to it.start.id.toString(), - "endId" to it.end.id.toString(), - ) - } - ) - val start = System.currentTimeMillis() - neo4jSession.executeWrite { tx -> - tx.run( - """ + this.chunked(edgeChunkSize).map { chunk -> + val b = Benchmark(Persistable::class.java, "Persisting chunk of ${chunk.size} edges") + val params = + mapOf( + "props" to + chunk.map { + mapOf( + "startId" to it.start.id.toString(), + "endId" to it.end.id.toString(), + "type" to it.label + ) + it.properties() + } + ) + neo4jSession.executeWrite { tx -> + tx.run( + """ UNWIND ${'$'}props AS map MATCH (s:Node {id: map.startId}) MATCH (e:Node {id: map.endId}) - CREATE (s)-[r:${it.key} {}]->(e) + WITH s, e, map, apoc.map.removeKeys(map, ['startId', 'endId', 'type']) AS properties + CALL apoc.create.relationship(s, map.type, properties, e) YIELD rel + RETURN rel """ - .trimIndent(), - params - ) - .consume() - } - log.info( - "Time Taken to process and save ${chunk.size} records to Neo4j Batch Insert took ${System.currentTimeMillis() - start} ms" - ) + .trimIndent(), + params + ) + .consume() } + b.stop() } } /** - * Returns the node's properties. This DOES NOT include relationships, but only properties directly - * attached to the node. + * Returns the [Persistable]'s properties. This DOES NOT include relationships, but only properties + * directly attached to the node/edge. */ -fun Node.properties(): Map { +fun Persistable.properties(): Map { val properties = mutableMapOf() - for (entry in schemaProperties) { + for (entry in this::class.schemaProperties) { val value = entry.value.call(this) if (value == null) { continue } - // TODO: generalize conversions - if (value is Name && entry.key == "name") { - properties += NameConverter().toGraphProperties(value) - } else if (value is Name) { - properties.put(entry.key, SimpleNameConverter().toGraphProperty(value)) - } else if (value is Uuid) { - properties.put(entry.key, value.toString()) - } else { - properties.put(entry.key, value) - } + value.convert(entry.key, properties) } return properties } -context(TransactionContext) -fun Edge<*>.persist() { - this@TransactionContext.run( - "MATCH (start { id: \$startId }), (end { id: \$endId } ) MERGE (start)-[r:${label} { }]->(end)", - mapOf("startId" to this.start.id.toString(), "endId" to this.end.id.toString()) - ) - .consume() -} - -val Node.labels: Set - get() { - val klazz = this::class - - // Check, if we already computed the labels for this node's class - return labelCache.computeIfAbsent(klazz) { setOf("Node", klazz.simpleName!!) } +/** + * Runs any conversions that are necessary by [CompositeAttributeConverter] and + * [org.neo4j.ogm.typeconversion.AttributeConverter]. Since both of these classes are Neo4J OGM + * classes, we need to find new base types at some point. + */ +fun Any.convert(originalKey: String, properties: MutableMap) { + // TODO: generalize conversions + if (this is Name && originalKey == "name") { + properties += NameConverter().toGraphProperties(this) + } else if (this is Name) { + properties.put(originalKey, SimpleNameConverter().toGraphProperty(this)) + } else if (this is Granularity) { + properties += DataflowGranularityConverter().toGraphProperties(this) + } else if (this is Enum<*>) { + properties.put(originalKey, this.name) + } else if (this is Uuid) { + properties.put(originalKey, this.toString()) + } else { + properties.put(originalKey, this) } +} val KClass.labels: Set get() { @@ -188,7 +195,7 @@ val KClass.labels: Set return labelCache.computeIfAbsent(this) { setOf("Node", this.simpleName!!) } } -val primitiveTypes = +val propertyTypes = setOf( String::class.createType(), Int::class.createType(), @@ -196,18 +203,18 @@ val primitiveTypes = Boolean::class.createType(), Name::class.createType(), Uuid::class.createType(), + Granularity::class.createType(), + DependenceType::class.createType(), ) -val Node.schemaProperties: Map> +val KClass.schemaProperties: Map> get() { - val klazz = this::class - // Check, if we already computed the labels for this node's class - return schemaPropertiesCache.computeIfAbsent(klazz) { - val schema = mutableMapOf>() + return schemaPropertiesCache.computeIfAbsent(this) { + val schema = mutableMapOf>() val properties = it.memberProperties for (property in properties) { - if (property.returnType.withNullability(false) in primitiveTypes) { + if (property.returnType.withNullability(false) in propertyTypes) { schema.put(property.name, property) } } From 7a86acada47a28c632996268e714a626bf83ec33 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 11 Dec 2024 10:01:48 +0100 Subject: [PATCH 09/25] Persisting language and scopes --- .../de/fraunhofer/aisec/cpg/graph/Node.kt | 2 +- .../fraunhofer/aisec/cpg/graph/Persistable.kt | 13 +++ .../fraunhofer/aisec/cpg/graph/edges/Edge.kt | 4 +- .../aisec/cpg/graph/scopes/Scope.kt | 8 +- .../de/fraunhofer/aisec/cpg/v2/Persistence.kt | 83 +++++++++++++------ 5 files changed, 81 insertions(+), 29 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt index 948be849757..52178600bdf 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt @@ -59,7 +59,7 @@ import org.slf4j.LoggerFactory /** The base class for all graph objects that are going to be persisted in the database. */ abstract class Node : IVisitable, - Persistable, + PersistedAsNode, LanguageProvider, ScopeProvider, ContextProvider, diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Persistable.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Persistable.kt index 01e0b33c363..fa44e2fb524 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Persistable.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Persistable.kt @@ -25,4 +25,17 @@ */ package de.fraunhofer.aisec.cpg.graph +import de.fraunhofer.aisec.cpg.graph.scopes.Scope + +/** + * This interface represents that the object will be persisted as a "node" in a graph database. This + * is not to be confused with our [Node] class -- for example a [Scope] is also a [PersistedAsNode], + * but not a [Node]. + */ +interface PersistedAsNode : Persistable + +/** This interface represents that the object will be persisted as a "edge" in a graph database. */ +interface PersistedAsEdge : Persistable + +/** This interface represents all objects that can be persisted in a graph database. */ interface Persistable diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt index 8b3d5dea424..0615dc2c1ee 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt @@ -29,7 +29,7 @@ import com.fasterxml.jackson.annotation.JsonBackReference import com.fasterxml.jackson.annotation.JsonIgnore import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.Node.Companion.TO_STRING_STYLE -import de.fraunhofer.aisec.cpg.graph.Persistable +import de.fraunhofer.aisec.cpg.graph.PersistedAsEdge import de.fraunhofer.aisec.cpg.graph.edges.flows.DependenceType import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.passes.ProgramDependenceGraphPass @@ -51,7 +51,7 @@ import org.neo4j.ogm.annotation.* * ``` */ @RelationshipEntity -abstract class Edge : Persistable, Cloneable { +abstract class Edge : PersistedAsEdge, Cloneable { /** Required field for object graph mapping. It contains the node id. */ @field:Id @field:GeneratedValue private val id: Long? = null diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt index cf322f7ff19..8e2b5cfe239 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt @@ -30,12 +30,14 @@ import de.fraunhofer.aisec.cpg.frontends.Language import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.Node.Companion.TO_STRING_STYLE +import de.fraunhofer.aisec.cpg.graph.PersistedAsNode import de.fraunhofer.aisec.cpg.graph.declarations.Declaration import de.fraunhofer.aisec.cpg.graph.declarations.ImportDeclaration import de.fraunhofer.aisec.cpg.graph.statements.LabelStatement import de.fraunhofer.aisec.cpg.graph.statements.LookupScopeStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference import de.fraunhofer.aisec.cpg.helpers.neo4j.NameConverter +import kotlin.uuid.Uuid import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.GeneratedValue import org.neo4j.ogm.annotation.Id @@ -60,10 +62,12 @@ sealed class Scope( @Relationship(value = "SCOPE", direction = Relationship.Direction.INCOMING) @JsonBackReference open var astNode: Node? -) { +) : PersistedAsNode { /** Required field for object graph mapping. It contains the scope id. */ - @Id @GeneratedValue var id: Long? = null + @Id @GeneratedValue var legacyId: Long? = null + + var id: Uuid = Uuid.random() /** FQN Name currently valid */ var scopedName: String? = null diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt index 26664a7533d..e4a443d8e23 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt @@ -29,6 +29,7 @@ import de.fraunhofer.aisec.cpg.TranslationResult import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.Persistable +import de.fraunhofer.aisec.cpg.graph.PersistedAsNode import de.fraunhofer.aisec.cpg.graph.edges.Edge import de.fraunhofer.aisec.cpg.graph.edges.allEdges import de.fraunhofer.aisec.cpg.graph.edges.flows.DependenceType @@ -62,7 +63,7 @@ val neo4jSession by lazy { driver.session() } -val labelCache: MutableMap, Set> = mutableMapOf() +val labelCache: MutableMap, Set> = mutableMapOf() val schemaPropertiesCache: MutableMap, Map>> = @@ -76,19 +77,30 @@ val nodeChunkSize = 10000 fun TranslationResult.persist() { val nodes = this@persist.nodes val edges = this@persist.allEdges>() + val scopes = this.finalCtx.scopeManager.filterScopes { true } + val languages = this.finalCtx.config.languages val b = Benchmark(Persistable::class.java, "Persisting translation result") - log.info("Persisting {} nodes", nodes.size) + log.info("Persisting {} AST nodes", nodes.size) nodes.persist() + log.info("Persisting {} scopes", nodes.size) + scopes.persist() + + log.info("Persisting {} languages", nodes.size) + languages.persist() + log.info("Persisting {} edges", edges.size) edges.persist() + log.info("Persisting {} extra relationships (language, scopes)", edges.size) + nodes.persistExtraRelationships() + b.stop() } -private fun List.persist() { +private fun List.persist() { this.chunked(nodeChunkSize).map { chunk -> val b = Benchmark(Persistable::class.java, "Persisting chunk of ${chunk.size} nodes") val params = @@ -117,21 +129,45 @@ private fun Collection>.persist() { } this.chunked(edgeChunkSize).map { chunk -> - val b = Benchmark(Persistable::class.java, "Persisting chunk of ${chunk.size} edges") - val params = - mapOf( - "props" to - chunk.map { - mapOf( - "startId" to it.start.id.toString(), - "endId" to it.end.id.toString(), - "type" to it.label - ) + it.properties() - } + createRelationships( + chunk.map { + mapOf( + "startId" to it.start.id.toString(), + "endId" to it.end.id.toString(), + "type" to it.label + ) + it.properties() + } + ) + } +} + +private fun List.persistExtraRelationships() { + this.flatMap { + listOf( + mapOf( + "startId" to it.id.toString(), + "endId" to it.language?.id.toString(), + "type" to "LANGUAGE" + ), + mapOf( + "startId" to it.id.toString(), + "endId" to it.scope?.id.toString(), + "type" to "SCOPE" + ), ) - neo4jSession.executeWrite { tx -> - tx.run( - """ + } + .chunked(10000) + .map { chunk -> createRelationships(chunk) } +} + +private fun createRelationships( + props: List>, +) { + val b = Benchmark(Persistable::class.java, "Persisting chunk of ${props.size} relationships") + val params = mapOf("props" to props) + neo4jSession.executeWrite { tx -> + tx.run( + """ UNWIND ${'$'}props AS map MATCH (s:Node {id: map.startId}) MATCH (e:Node {id: map.endId}) @@ -139,13 +175,12 @@ private fun Collection>.persist() { CALL apoc.create.relationship(s, map.type, properties, e) YIELD rel RETURN rel """ - .trimIndent(), - params - ) - .consume() - } - b.stop() + .trimIndent(), + params + ) + .consume() } + b.stop() } /** @@ -189,7 +224,7 @@ fun Any.convert(originalKey: String, properties: MutableMap) { } } -val KClass.labels: Set +val KClass.labels: Set get() { // Check, if we already computed the labels for this node's class return labelCache.computeIfAbsent(this) { setOf("Node", this.simpleName!!) } From 8f895546c318696b7a15f49438c2f08280951da9 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 11 Dec 2024 12:54:58 +0100 Subject: [PATCH 10/25] Scopes are now nodes --- .../de/fraunhofer/aisec/cpg/graph/Node.kt | 2 +- .../fraunhofer/aisec/cpg/graph/Persistable.kt | 12 ---- .../fraunhofer/aisec/cpg/graph/edges/Edge.kt | 4 +- .../aisec/cpg/graph/scopes/NameScope.kt | 3 +- .../aisec/cpg/graph/scopes/Scope.kt | 17 +---- .../de/fraunhofer/aisec/cpg/v2/Persistence.kt | 65 +++++++++++++------ 6 files changed, 52 insertions(+), 51 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt index 52178600bdf..948be849757 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt @@ -59,7 +59,7 @@ import org.slf4j.LoggerFactory /** The base class for all graph objects that are going to be persisted in the database. */ abstract class Node : IVisitable, - PersistedAsNode, + Persistable, LanguageProvider, ScopeProvider, ContextProvider, diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Persistable.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Persistable.kt index fa44e2fb524..604bbb71966 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Persistable.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Persistable.kt @@ -25,17 +25,5 @@ */ package de.fraunhofer.aisec.cpg.graph -import de.fraunhofer.aisec.cpg.graph.scopes.Scope - -/** - * This interface represents that the object will be persisted as a "node" in a graph database. This - * is not to be confused with our [Node] class -- for example a [Scope] is also a [PersistedAsNode], - * but not a [Node]. - */ -interface PersistedAsNode : Persistable - -/** This interface represents that the object will be persisted as a "edge" in a graph database. */ -interface PersistedAsEdge : Persistable - /** This interface represents all objects that can be persisted in a graph database. */ interface Persistable diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt index 0615dc2c1ee..8b3d5dea424 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt @@ -29,7 +29,7 @@ import com.fasterxml.jackson.annotation.JsonBackReference import com.fasterxml.jackson.annotation.JsonIgnore import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.Node.Companion.TO_STRING_STYLE -import de.fraunhofer.aisec.cpg.graph.PersistedAsEdge +import de.fraunhofer.aisec.cpg.graph.Persistable import de.fraunhofer.aisec.cpg.graph.edges.flows.DependenceType import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.passes.ProgramDependenceGraphPass @@ -51,7 +51,7 @@ import org.neo4j.ogm.annotation.* * ``` */ @RelationshipEntity -abstract class Edge : PersistedAsEdge, Cloneable { +abstract class Edge : Persistable, Cloneable { /** Required field for object graph mapping. It contains the node id. */ @field:Id @field:GeneratedValue private val id: Long? = null diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/NameScope.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/NameScope.kt index 4280e774b99..b70f98b05da 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/NameScope.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/NameScope.kt @@ -25,6 +25,7 @@ */ package de.fraunhofer.aisec.cpg.graph.scopes +import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.Node /** @@ -37,6 +38,6 @@ sealed class NameScope(node: Node?) : StructureDeclarationScope(node) { init { astNode = node // Set the name so that we can use it as a namespace later - name = node?.name + name = node?.name ?: Name(EMPTY_NAME) } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt index 8e2b5cfe239..d393e137a5d 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/scopes/Scope.kt @@ -27,23 +27,16 @@ package de.fraunhofer.aisec.cpg.graph.scopes import com.fasterxml.jackson.annotation.JsonBackReference import de.fraunhofer.aisec.cpg.frontends.Language -import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.Node.Companion.TO_STRING_STYLE -import de.fraunhofer.aisec.cpg.graph.PersistedAsNode import de.fraunhofer.aisec.cpg.graph.declarations.Declaration import de.fraunhofer.aisec.cpg.graph.declarations.ImportDeclaration import de.fraunhofer.aisec.cpg.graph.statements.LabelStatement import de.fraunhofer.aisec.cpg.graph.statements.LookupScopeStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference -import de.fraunhofer.aisec.cpg.helpers.neo4j.NameConverter -import kotlin.uuid.Uuid import org.apache.commons.lang3.builder.ToStringBuilder -import org.neo4j.ogm.annotation.GeneratedValue -import org.neo4j.ogm.annotation.Id import org.neo4j.ogm.annotation.NodeEntity import org.neo4j.ogm.annotation.Relationship -import org.neo4j.ogm.annotation.typeconversion.Convert /** * A symbol is a simple, local name. It is valid within the scope that declares it and all of its @@ -62,19 +55,11 @@ sealed class Scope( @Relationship(value = "SCOPE", direction = Relationship.Direction.INCOMING) @JsonBackReference open var astNode: Node? -) : PersistedAsNode { - - /** Required field for object graph mapping. It contains the scope id. */ - @Id @GeneratedValue var legacyId: Long? = null - - var id: Uuid = Uuid.random() +) : Node() { /** FQN Name currently valid */ var scopedName: String? = null - /** The real new name */ - @Convert(NameConverter::class) var name: Name? = null - /** * Scopes are nested and therefore have a parent child relationship, this two members will help * navigate through the scopes,e.g. when looking up variables. diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt index e4a443d8e23..e4c714f7a0e 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt @@ -29,21 +29,22 @@ import de.fraunhofer.aisec.cpg.TranslationResult import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.Persistable -import de.fraunhofer.aisec.cpg.graph.PersistedAsNode import de.fraunhofer.aisec.cpg.graph.edges.Edge import de.fraunhofer.aisec.cpg.graph.edges.allEdges import de.fraunhofer.aisec.cpg.graph.edges.flows.DependenceType import de.fraunhofer.aisec.cpg.graph.edges.flows.Granularity import de.fraunhofer.aisec.cpg.graph.nodes +import de.fraunhofer.aisec.cpg.graph.types.HasType +import de.fraunhofer.aisec.cpg.graph.types.SecondOrderType import de.fraunhofer.aisec.cpg.helpers.Benchmark import de.fraunhofer.aisec.cpg.helpers.neo4j.DataflowGranularityConverter import de.fraunhofer.aisec.cpg.helpers.neo4j.NameConverter import de.fraunhofer.aisec.cpg.helpers.neo4j.SimpleNameConverter -import kotlin.collections.plusAssign import kotlin.reflect.KClass import kotlin.reflect.KProperty1 import kotlin.reflect.full.createType import kotlin.reflect.full.memberProperties +import kotlin.reflect.full.superclasses import kotlin.reflect.full.withNullability import kotlin.uuid.Uuid import org.neo4j.driver.GraphDatabase @@ -63,7 +64,7 @@ val neo4jSession by lazy { driver.session() } -val labelCache: MutableMap, Set> = mutableMapOf() +val labelCache: MutableMap, Set> = mutableMapOf() val schemaPropertiesCache: MutableMap, Map>> = @@ -75,32 +76,36 @@ val edgeChunkSize = 10000 val nodeChunkSize = 10000 fun TranslationResult.persist() { - val nodes = this@persist.nodes - val edges = this@persist.allEdges>() + val b = Benchmark(Persistable::class.java, "Persisting translation result") + + val astNodes = this@persist.nodes val scopes = this.finalCtx.scopeManager.filterScopes { true } val languages = this.finalCtx.config.languages + val types = + this.finalCtx.typeManager.firstOrderTypes + this.finalCtx.typeManager.secondOrderTypes + val nodes = listOf(astNodes, scopes, languages, types).flatten() + val edges = this@persist.allEdges>() - val b = Benchmark(Persistable::class.java, "Persisting translation result") - - log.info("Persisting {} AST nodes", nodes.size) + log.info( + "Persisting {} nodes: AST nodes ({}), types ({}), scopes ({}) and languages ({})", + nodes.size, + astNodes.size, + types.size, + scopes.size, + languages.size + ) nodes.persist() - log.info("Persisting {} scopes", nodes.size) - scopes.persist() - - log.info("Persisting {} languages", nodes.size) - languages.persist() - log.info("Persisting {} edges", edges.size) edges.persist() - log.info("Persisting {} extra relationships (language, scopes)", edges.size) + log.info("Persisting {} extra relationships (types, scopes, languages)", edges.size) nodes.persistExtraRelationships() b.stop() } -private fun List.persist() { +private fun List.persist() { this.chunked(nodeChunkSize).map { chunk -> val b = Benchmark(Persistable::class.java, "Persisting chunk of ${chunk.size} nodes") val params = @@ -141,9 +146,13 @@ private fun Collection>.persist() { } } +/** + * Some of our relationships are not real "edges" (yet). We need to handle these case separately + * (for now). + */ private fun List.persistExtraRelationships() { this.flatMap { - listOf( + listOfNotNull( mapOf( "startId" to it.id.toString(), "endId" to it.language?.id.toString(), @@ -154,6 +163,21 @@ private fun List.persistExtraRelationships() { "endId" to it.scope?.id.toString(), "type" to "SCOPE" ), + if (it is HasType) { + mapOf( + "startId" to it.id.toString(), + "endId" to it.type.id.toString(), + "type" to "TYPE" + ) + } else if (it is SecondOrderType) { + mapOf( + "startId" to it.id.toString(), + "endId" to it.elementType.id.toString(), + "type" to "ELEMENT_TYPE" + ) + } else { + null + } ) } .chunked(10000) @@ -224,10 +248,13 @@ fun Any.convert(originalKey: String, properties: MutableMap) { } } -val KClass.labels: Set +val KClass.labels: Set get() { // Check, if we already computed the labels for this node's class - return labelCache.computeIfAbsent(this) { setOf("Node", this.simpleName!!) } + return labelCache.computeIfAbsent(this) { + setOfNotNull("Node", this.simpleName) + + it.superclasses.mapNotNull { it.simpleName } + } } val propertyTypes = From e3a9e241cc46483045871188f925c865de1e65b2 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 11 Dec 2024 14:30:06 +0100 Subject: [PATCH 11/25] Correct label hierarchy --- .../de/fraunhofer/aisec/cpg/v2/Persistence.kt | 51 ++++++++++++++----- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt index e4c714f7a0e..8e7d4f9cb96 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt @@ -34,12 +34,14 @@ import de.fraunhofer.aisec.cpg.graph.edges.allEdges import de.fraunhofer.aisec.cpg.graph.edges.flows.DependenceType import de.fraunhofer.aisec.cpg.graph.edges.flows.Granularity import de.fraunhofer.aisec.cpg.graph.nodes +import de.fraunhofer.aisec.cpg.graph.types.FunctionPointerType import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.SecondOrderType import de.fraunhofer.aisec.cpg.helpers.Benchmark import de.fraunhofer.aisec.cpg.helpers.neo4j.DataflowGranularityConverter import de.fraunhofer.aisec.cpg.helpers.neo4j.NameConverter import de.fraunhofer.aisec.cpg.helpers.neo4j.SimpleNameConverter +import de.fraunhofer.aisec.cpg.helpers.toIdentitySet import kotlin.reflect.KClass import kotlin.reflect.KProperty1 import kotlin.reflect.full.createType @@ -64,7 +66,7 @@ val neo4jSession by lazy { driver.session() } -val labelCache: MutableMap, Set> = mutableMapOf() +val labelCache: MutableMap, Set> = mutableMapOf() val schemaPropertiesCache: MutableMap, Map>> = @@ -82,7 +84,8 @@ fun TranslationResult.persist() { val scopes = this.finalCtx.scopeManager.filterScopes { true } val languages = this.finalCtx.config.languages val types = - this.finalCtx.typeManager.firstOrderTypes + this.finalCtx.typeManager.secondOrderTypes + (this.finalCtx.typeManager.firstOrderTypes + this.finalCtx.typeManager.secondOrderTypes) + .toIdentitySet() val nodes = listOf(astNodes, scopes, languages, types).flatten() val edges = this@persist.allEdges>() @@ -99,7 +102,7 @@ fun TranslationResult.persist() { log.info("Persisting {} edges", edges.size) edges.persist() - log.info("Persisting {} extra relationships (types, scopes, languages)", edges.size) + log.info("Persisting extra relationships (types, scopes, languages)") nodes.persistExtraRelationships() b.stop() @@ -147,11 +150,12 @@ private fun Collection>.persist() { } /** - * Some of our relationships are not real "edges" (yet). We need to handle these case separately - * (for now). + * Some of our relationships are not real "edges" (i.e., [Edge]) (yet). We need to handle these case + * separately (for now). */ private fun List.persistExtraRelationships() { - this.flatMap { + val relationships = + this.flatMap { listOfNotNull( mapOf( "startId" to it.id.toString(), @@ -175,13 +179,19 @@ private fun List.persistExtraRelationships() { "endId" to it.elementType.id.toString(), "type" to "ELEMENT_TYPE" ) + } else if (it is FunctionPointerType) { + mapOf( + "startId" to it.id.toString(), + "endId" to it.returnType.id.toString(), + "type" to "RETURN_TYPE" + ) } else { null } ) } - .chunked(10000) - .map { chunk -> createRelationships(chunk) } + + relationships.chunked(10000).map { chunk -> createRelationships(chunk) } } private fun createRelationships( @@ -248,13 +258,28 @@ fun Any.convert(originalKey: String, properties: MutableMap) { } } -val KClass.labels: Set +val KClass<*>.labels: Set get() { - // Check, if we already computed the labels for this node's class - return labelCache.computeIfAbsent(this) { - setOfNotNull("Node", this.simpleName) + - it.superclasses.mapNotNull { it.simpleName } + // Ignore interfaces and the Kotlin base class + if (this.java.isInterface || this == Any::class) { + return setOf() } + + val cacheKey = this + + // Note: we cannot use computeIfAbsent here, because we are calling our function + // recursively and this would result in a ConcurrentModificationException + if (labelCache.containsKey(cacheKey)) { + return labelCache[cacheKey] ?: setOf() + } + + val labels = mutableSetOf() + labels.addAll(this.superclasses.flatMap { it.labels }) + this.simpleName?.let { labels.add(it) } + + // update the cache + labelCache[cacheKey] = labels + return labels } val propertyTypes = From 5a8c04572ffaf3b778e68cf9dba3593d85461197 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 11 Dec 2024 14:51:01 +0100 Subject: [PATCH 12/25] Pseudo-support for multi-labels --- .../de/fraunhofer/aisec/cpg/graph/Assignment.kt | 2 +- .../fraunhofer/aisec/cpg/graph/MermaidPrinter.kt | 2 +- .../de/fraunhofer/aisec/cpg/graph/edges/Edge.kt | 2 +- .../aisec/cpg/graph/edges/ast/AstEdge.kt | 16 ++++++++-------- .../cpg/graph/edges/ast/TemplateArgument.kt | 2 +- .../cpg/graph/edges/flows/ControlDependence.kt | 2 +- .../aisec/cpg/graph/edges/flows/Dataflow.kt | 4 ++-- .../cpg/graph/edges/flows/EvaluationOrder.kt | 2 +- .../aisec/cpg/graph/edges/flows/Invoke.kt | 2 +- .../aisec/cpg/graph/edges/flows/Usage.kt | 2 +- .../statements/expressions/CallExpression.kt | 2 +- .../de/fraunhofer/aisec/cpg/v2/Persistence.kt | 16 ++++++++++------ 12 files changed, 29 insertions(+), 25 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Assignment.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Assignment.kt index 6cc946d0d16..d20a0c82e58 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Assignment.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Assignment.kt @@ -46,5 +46,5 @@ class Assignment( /** The holder of this assignment */ @JsonIgnore val holder: AssignmentHolder ) : Edge(value, target as Node) { - override var label: String = "ASSIGMENT" + override var labels = setOf("ASSIGMENT") } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/MermaidPrinter.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/MermaidPrinter.kt index 6f04d5dd1b1..1da2d709ce1 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/MermaidPrinter.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/MermaidPrinter.kt @@ -109,7 +109,7 @@ fun > Node.printGraph( private fun Edge.label(): String { val builder = StringBuilder() builder.append("\"") - builder.append(this.label) + builder.append(this.labels) if (this is Dataflow) { var granularity = this.granularity diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt index 8b3d5dea424..064b0caa29d 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/Edge.kt @@ -71,7 +71,7 @@ abstract class Edge : Persistable, Cloneable { end = edge.end } - abstract var label: String + abstract var labels: Set /** * The index of this node, if it is stored in an diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt index a83d7ed01db..5e7d09dce9d 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt @@ -33,7 +33,7 @@ import org.neo4j.ogm.annotation.* /** This property edge describes a parent/child relationship in the Abstract Syntax Tree (AST). */ @RelationshipEntity -open class AstEdge(start: Node, end: T, override var label: String = "AST") : +open class AstEdge(start: Node, end: T, override var labels: Set = setOf("AST")) : Edge(start, end) { init { end.astParent = start @@ -42,7 +42,7 @@ open class AstEdge(start: Node, end: T, override var label: String = " /** Creates an [AstEdges] container starting from this node. */ fun Node.astEdgesOf( - label: String = "AST", + label: String? = null, onAdd: ((AstEdge) -> Unit)? = null, onRemove: ((AstEdge) -> Unit)? = null, ): AstEdges> { @@ -54,12 +54,12 @@ fun Node.astEdgesOf( * container). */ fun Node.astOptionalEdgeOf( - label: String = "AST", + label: String? = null, onChanged: ((old: AstEdge?, new: AstEdge?) -> Unit)? = null ): EdgeSingletonList> { return EdgeSingletonList( thisRef = this, - init = { start, end -> AstEdge(start, end, label = label) }, + init = { start, end -> AstEdge(start, end, labels = setOfNotNull(label, "AST")) }, outgoing = true, onChanged = onChanged, of = null @@ -71,12 +71,12 @@ fun Node.astOptionalEdgeOf( */ fun Node.astEdgeOf( of: NodeType, - label: String, + label: String? = null, onChanged: ((old: AstEdge?, new: AstEdge?) -> Unit)? = null, ): EdgeSingletonList> { return EdgeSingletonList( thisRef = this, - init = { start, end -> AstEdge(start, end, label = label) }, + init = { start, end -> AstEdge(start, end, labels = setOfNotNull(label, "AST")) }, outgoing = true, onChanged = onChanged, of = of @@ -86,12 +86,12 @@ fun Node.astEdgeOf( /** This property edge list describes elements that are AST children of a node. */ open class AstEdges>( thisRef: Node, - label: String = "AST", + label: String? = null, onAdd: ((PropertyEdgeType) -> Unit)? = null, onRemove: ((PropertyEdgeType) -> Unit)? = null, @Suppress("UNCHECKED_CAST") init: (start: Node, end: NodeType) -> PropertyEdgeType = { start, end -> - AstEdge(start, end, label = label) as PropertyEdgeType + AstEdge(start, end, labels = setOfNotNull(label, "AST")) as PropertyEdgeType }, ) : EdgeList( diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/TemplateArgument.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/TemplateArgument.kt index b06c3bdfd19..46374d959c9 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/TemplateArgument.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/TemplateArgument.kt @@ -35,7 +35,7 @@ class TemplateArgument( end: NodeType, label: String, var instantiation: TemplateInitialization? = TemplateInitialization.EXPLICIT, -) : AstEdge(start, end, label) +) : AstEdge(start, end, setOf(label, "AST")) /** A container for [TemplateArgument] edges. */ class TemplateArguments(thisRef: Node, label: String) : diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ControlDependence.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ControlDependence.kt index 3591e0eaf56..5f3badedad5 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ControlDependence.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/ControlDependence.kt @@ -49,7 +49,7 @@ class ControlDependence( dependence = DependenceType.CONTROL } - override var label: String = "CDG" + override var labels = setOf("CDG") override fun equals(other: Any?): Boolean { if (this === other) return true diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Dataflow.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Dataflow.kt index 4ce9ff4ebdb..07ba1dc1047 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Dataflow.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Dataflow.kt @@ -93,7 +93,7 @@ open class Dataflow( @JsonIgnore var granularity: Granularity = default() ) : Edge(start, end) { - override var label: String = "DFG" + override var labels = setOf("DFG") override fun equals(other: Any?): Boolean { if (this === other) return true @@ -134,7 +134,7 @@ class ContextSensitiveDataflow( val callingContext: CallingContext ) : Dataflow(start, end, granularity) { - override var label: String = "DFG" + override var labels = setOf("DFG") override fun equals(other: Any?): Boolean { if (this === other) return true diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/EvaluationOrder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/EvaluationOrder.kt index e6af5e8d818..7ac7126b3bf 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/EvaluationOrder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/EvaluationOrder.kt @@ -69,7 +69,7 @@ class EvaluationOrder( return result } - override var label: String = "EOG" + override var labels = setOf("EOG") } /** diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Invoke.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Invoke.kt index e6a94996835..9bbaffad821 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Invoke.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Invoke.kt @@ -43,7 +43,7 @@ class Invoke( */ var dynamicInvoke: Boolean = false, ) : Edge(start, end) { - override var label: String = "INVOKES" + override var labels = setOf("INVOKES") override fun equals(other: Any?): Boolean { if (this === other) return true diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Usage.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Usage.kt index b4a41a6821a..0178456adfb 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Usage.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/flows/Usage.kt @@ -53,7 +53,7 @@ class Usage( return result } - override var label: String = "USAGE" + override var labels = setOf("USAGE") } /** A container for [Usage] edges. [NodeType] is necessary because of the Neo4J OGM. */ diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt index 218ae9d250d..e33affe188e 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt @@ -121,7 +121,7 @@ open class CallExpression : /** Adds the specified [expression] with an optional [name] to this call. */ fun addArgument(expression: Expression, name: String? = null) { - val edge = AstEdge(this, expression, label = "ARGUMENTS") + val edge = AstEdge(this, expression, labels = setOf("ARGUMENTS", "ARGUMENT")) edge.name = name argumentEdges.add(edge) diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt index 8e7d4f9cb96..d5b01e3d03b 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt @@ -138,12 +138,16 @@ private fun Collection>.persist() { this.chunked(edgeChunkSize).map { chunk -> createRelationships( - chunk.map { - mapOf( - "startId" to it.start.id.toString(), - "endId" to it.end.id.toString(), - "type" to it.label - ) + it.properties() + chunk.flatMap { edge -> + // Since Neo4J does not support multiple labels on edges, but we do internally, we + // duplicate the edge for each label + edge.labels.map { label -> + mapOf( + "startId" to edge.start.id.toString(), + "endId" to edge.end.id.toString(), + "type" to label + ) + edge.properties() + } } ) } From 7cfa6952eaf19a09e7136d9e87c18edb5ba2b73b Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 11 Dec 2024 15:01:18 +0100 Subject: [PATCH 13/25] Using new API in neo4j application --- .../cpg/{v2 => persistence}/Persistence.kt | 27 +++---- .../aisec/cpg_vis_neo4j/Application.kt | 79 ++++--------------- .../aisec/cpg/v2/TestPersistence.kt | 41 ---------- .../aisec/cpg_vis_neo4j/Neo4JTest.kt | 35 -------- 4 files changed, 26 insertions(+), 156 deletions(-) rename cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/{v2 => persistence}/Persistence.kt (95%) delete mode 100644 cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg/v2/TestPersistence.kt diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Persistence.kt similarity index 95% rename from cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt rename to cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Persistence.kt index d5b01e3d03b..fd264b40ceb 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/v2/Persistence.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Persistence.kt @@ -23,7 +23,7 @@ * \______/ \__| \______/ * */ -package de.fraunhofer.aisec.cpg.v2 +package de.fraunhofer.aisec.cpg.persistence import de.fraunhofer.aisec.cpg.TranslationResult import de.fraunhofer.aisec.cpg.graph.Name @@ -49,7 +49,7 @@ import kotlin.reflect.full.memberProperties import kotlin.reflect.full.superclasses import kotlin.reflect.full.withNullability import kotlin.uuid.Uuid -import org.neo4j.driver.GraphDatabase +import org.neo4j.driver.Session import org.neo4j.ogm.typeconversion.CompositeAttributeConverter import org.slf4j.LoggerFactory @@ -57,15 +57,6 @@ import org.slf4j.LoggerFactory * docker run \ --name neo4j-apoc \ -p 7474:7474 -p 7687:7687 \ -d \ -e NEO4J_AUTH=neo4j/password \ * -e NEO4JLABS_PLUGINS='["apoc"]' \ neo4j:5 */ -val dbUri = "neo4j://localhost" -val dbUser = "neo4j" -val dbPassword = "password" - -val neo4jSession by lazy { - val driver = GraphDatabase.driver(dbUri, org.neo4j.driver.AuthTokens.basic(dbUser, dbPassword)) - driver.session() -} - val labelCache: MutableMap, Set> = mutableMapOf() val schemaPropertiesCache: @@ -77,6 +68,7 @@ val log = LoggerFactory.getLogger("Persistence") val edgeChunkSize = 10000 val nodeChunkSize = 10000 +context(Session) fun TranslationResult.persist() { val b = Benchmark(Persistable::class.java, "Persisting translation result") @@ -108,12 +100,13 @@ fun TranslationResult.persist() { b.stop() } +context(Session) private fun List.persist() { this.chunked(nodeChunkSize).map { chunk -> val b = Benchmark(Persistable::class.java, "Persisting chunk of ${chunk.size} nodes") val params = mapOf("props" to chunk.map { mapOf("labels" to it::class.labels) + it.properties() }) - neo4jSession.executeWrite { tx -> + this@Session.executeWrite { tx -> tx.run( """ UNWIND ${"$"}props AS map @@ -129,10 +122,11 @@ private fun List.persist() { } } +context(Session) private fun Collection>.persist() { // Create an index for the "id" field of node, because we are "MATCH"ing on it in the edge // creation. We need to wait for this to be finished - neo4jSession.executeWrite { tx -> + this@Session.executeWrite { tx -> tx.run("CREATE INDEX IF NOT EXISTS FOR (n:Node) ON (n.id)").consume() } @@ -157,6 +151,7 @@ private fun Collection>.persist() { * Some of our relationships are not real "edges" (i.e., [Edge]) (yet). We need to handle these case * separately (for now). */ +context(Session) private fun List.persistExtraRelationships() { val relationships = this.flatMap { @@ -195,15 +190,15 @@ private fun List.persistExtraRelationships() { ) } - relationships.chunked(10000).map { chunk -> createRelationships(chunk) } + relationships.chunked(10000).map { chunk -> this@Session.createRelationships(chunk) } } -private fun createRelationships( +private fun Session.createRelationships( props: List>, ) { val b = Benchmark(Persistable::class.java, "Persisting chunk of ${props.size} relationships") val params = mapOf("props" to props) - neo4jSession.executeWrite { tx -> + executeWrite { tx -> tx.run( """ UNWIND ${'$'}props AS map diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt index ea2b2257515..b286dfc02a5 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt @@ -30,23 +30,20 @@ import de.fraunhofer.aisec.cpg.* import de.fraunhofer.aisec.cpg.frontends.CompilationDatabase.Companion.fromFile import de.fraunhofer.aisec.cpg.helpers.Benchmark import de.fraunhofer.aisec.cpg.passes.* +import de.fraunhofer.aisec.cpg.persistence.persist import java.io.File import java.net.ConnectException import java.nio.file.Paths import java.util.concurrent.Callable import kotlin.reflect.KClass import kotlin.system.exitProcess -import org.neo4j.driver.exceptions.AuthenticationException -import org.neo4j.ogm.config.Configuration +import org.neo4j.driver.GraphDatabase import org.neo4j.ogm.context.EntityGraphMapper import org.neo4j.ogm.context.MappingContext import org.neo4j.ogm.cypher.compiler.MultiStatementCypherCompiler import org.neo4j.ogm.cypher.compiler.builders.node.DefaultNodeBuilder import org.neo4j.ogm.cypher.compiler.builders.node.DefaultRelationshipBuilder -import org.neo4j.ogm.exception.ConnectionException import org.neo4j.ogm.metadata.MetaData -import org.neo4j.ogm.session.Session -import org.neo4j.ogm.session.SessionFactory import org.slf4j.Logger import org.slf4j.LoggerFactory import picocli.CommandLine @@ -381,33 +378,14 @@ class Application : Callable { * Pushes the whole translationResult to the neo4j db. * * @param translationResult, not null - * @throws InterruptedException, if the thread is interrupted while it try´s to connect to the - * neo4j db. - * @throws ConnectException, if there is no connection to bolt://localhost:7687 possible */ - @Throws(InterruptedException::class, ConnectException::class) fun pushToNeo4j(translationResult: TranslationResult) { - val bench = Benchmark(this.javaClass, "Push cpg to neo4j", false, translationResult) - log.info("Using import depth: $depth") - log.info( - "Count base nodes to save: " + - translationResult.components.size + - translationResult.additionalNodes.size - ) - - val sessionAndSessionFactoryPair = connect() - - val session = sessionAndSessionFactoryPair.first - session.beginTransaction().use { transaction -> - if (!noPurgeDb) session.purgeDatabase() - session.save(translationResult.components, depth) - session.save(translationResult.additionalNodes, depth) - transaction.commit() + val session = connect() + with(session) { + if (!noPurgeDb) executeWrite { tx -> tx.run("MATCH (n) DETACH DELETE n").consume() } + translationResult.persist() } - - session.clear() - sessionAndSessionFactoryPair.second.close() - bench.addMeasurement() + session.close() } /** @@ -420,41 +398,14 @@ class Application : Callable { * @throws ConnectException, if there is no connection to bolt://localhost:7687 possible */ @Throws(InterruptedException::class, ConnectException::class) - fun connect(): Pair { - var fails = 0 - var sessionFactory: SessionFactory? = null - var session: Session? = null - while (session == null && fails < MAX_COUNT_OF_FAILS) { - try { - val configuration = - Configuration.Builder() - .uri("$PROTOCOL$host:$port") - .credentials(neo4jUsername, neo4jPassword) - .verifyConnection(VERIFY_CONNECTION) - .build() - sessionFactory = SessionFactory(configuration, *packages) - - session = sessionFactory.openSession() - } catch (ex: ConnectionException) { - sessionFactory = null - fails++ - log.error( - "Unable to connect to localhost:7687, " + - "ensure the database is running and that " + - "there is a working network connection to it." - ) - Thread.sleep(TIME_BETWEEN_CONNECTION_TRIES) - } catch (ex: AuthenticationException) { - log.error("Unable to connect to localhost:7687, wrong username/password!") - exitProcess(EXIT_FAILURE) - } - } - if (session == null || sessionFactory == null) { - log.error("Unable to connect to localhost:7687") - exitProcess(EXIT_FAILURE) - } - assert(fails <= MAX_COUNT_OF_FAILS) - return Pair(session, sessionFactory) + fun connect(): org.neo4j.driver.Session { + val driver = + GraphDatabase.driver( + "$PROTOCOL$host:$port", + org.neo4j.driver.AuthTokens.basic(neo4jUsername, neo4jPassword) + ) + driver.verifyConnectivity() + return driver.session() } /** diff --git a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg/v2/TestPersistence.kt b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg/v2/TestPersistence.kt deleted file mode 100644 index 9ec02f48961..00000000000 --- a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg/v2/TestPersistence.kt +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * $$$$$$\ $$$$$$$\ $$$$$$\ - * $$ __$$\ $$ __$$\ $$ __$$\ - * $$ / \__|$$ | $$ |$$ / \__| - * $$ | $$$$$$$ |$$ |$$$$\ - * $$ | $$ ____/ $$ |\_$$ | - * $$ | $$\ $$ | $$ | $$ | - * \$$$$$ |$$ | \$$$$$ | - * \______/ \__| \______/ - * - */ -package de.fraunhofer.aisec.cpg.v2 - -import de.fraunhofer.aisec.cpg_vis_neo4j.createTranslationResult -import org.junit.jupiter.api.Tag -import org.junit.jupiter.api.Test - -@Tag("integration") -class TestPersistence { - @Test - fun testPersist() { - val result = createTranslationResult() - - neo4jSession.executeWrite { tx -> tx.run("MATCH (n) DETACH DELETE n").consume() } - result.second.persist() - } -} diff --git a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt index 96f477d58b6..a77d2d0269c 100644 --- a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt +++ b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt @@ -25,14 +25,9 @@ */ package de.fraunhofer.aisec.cpg_vis_neo4j -import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend -import de.fraunhofer.aisec.cpg.graph.Name -import de.fraunhofer.aisec.cpg.graph.builder.translationResult -import de.fraunhofer.aisec.cpg.graph.declarations.ImportDeclaration import de.fraunhofer.aisec.cpg.graph.functions import kotlin.test.Test import kotlin.test.assertEquals -import kotlin.test.assertNotNull import org.junit.jupiter.api.Tag @Tag("integration") @@ -47,34 +42,4 @@ class Neo4JTest { application.pushToNeo4j(translationResult) } - - @Test - fun testSimpleNameConverter() { - val result = - with(TestLanguageFrontend()) { - translationResult { - val import = ImportDeclaration() - import.name = Name("myname") - import.alias = Name("myname", Name("myparent"), "::") - additionalNodes += import - } - } - - val app = Application() - app.pushToNeo4j(result) - - val sessionAndSessionFactoryPair = app.connect() - - val session = sessionAndSessionFactoryPair.first - session.beginTransaction().use { transaction -> - val imports = session.loadAll(ImportDeclaration::class.java) - assertNotNull(imports) - - var loadedImport = imports.singleOrNull() - assertNotNull(loadedImport) - assertEquals("myname", loadedImport.alias?.localName) - - transaction.commit() - } - } } From b330d57d462748a309757dec7e82314efa9dae7a Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 11 Dec 2024 15:18:35 +0100 Subject: [PATCH 14/25] Cleanup and documentation --- .github/workflows/build.yml | 2 +- .../aisec/cpg/persistence/Common.kt | 187 +++++++++++++++++ .../persistence/{Persistence.kt => Neo4J.kt} | 193 +++++++----------- .../aisec/cpg_vis_neo4j/Application.kt | 14 +- 4 files changed, 272 insertions(+), 124 deletions(-) create mode 100644 cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt rename cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/{Persistence.kt => Neo4J.kt} (61%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 785475de5cc..31eb168719d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,7 +38,7 @@ jobs: go-version: 1.21 - name: Setup neo4j run: | - docker run -d --env NEO4J_AUTH=neo4j/password -p7474:7474 -p7687:7687 neo4j || true + docker run -d --env NEO4J_AUTH=neo4j/password -p7474:7474 -p7687:7687 -e NEO4JLABS_PLUGINS='["apoc"]' neo4j:5 || true - name: Determine Version run: | # determine version from tag diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt new file mode 100644 index 00000000000..df0f2e29f33 --- /dev/null +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.persistence + +import de.fraunhofer.aisec.cpg.graph.Name +import de.fraunhofer.aisec.cpg.graph.Persistable +import de.fraunhofer.aisec.cpg.graph.edges.flows.DependenceType +import de.fraunhofer.aisec.cpg.graph.edges.flows.Granularity +import de.fraunhofer.aisec.cpg.helpers.neo4j.DataflowGranularityConverter +import de.fraunhofer.aisec.cpg.helpers.neo4j.NameConverter +import de.fraunhofer.aisec.cpg.helpers.neo4j.SimpleNameConverter +import kotlin.reflect.KClass +import kotlin.reflect.KProperty1 +import kotlin.reflect.full.createType +import kotlin.reflect.full.memberProperties +import kotlin.reflect.full.superclasses +import kotlin.reflect.full.withNullability +import kotlin.uuid.Uuid +import org.neo4j.ogm.typeconversion.CompositeAttributeConverter +import org.slf4j.LoggerFactory + +/** + * A cache used to store and retrieve sets of labels associated with specific Kotlin class types. + * + * This mutable map uses a Kotlin class type as the key and a set of strings representing associated + * labels as the value. The [labelCache] provides efficient lookup and prevents redundant + * re-computation of labels for the same class type. + */ +val labelCache: MutableMap, Set> = mutableMapOf() + +/** + * A cache mapping classes of type [Persistable] to their respective properties. + * + * This mutable map stores metadata about [Persistable] objects. For each specific class that + * implements the [Persistable] interface, it caches a mapping between property names and their + * corresponding [KProperty1] references. This allows efficient reflection-based access to class + * properties without repeatedly inspecting the class at runtime. + * + * The key in the map is the [KClass] of a subclass of [Persistable]. The value is a [Map] where the + * keys are strings representing the property names, and the values are [KProperty1] references + * pointing to those properties. This can be used for dynamic property access or serialization + * processes. + */ +val schemaPropertiesCache: + MutableMap, Map>> = + mutableMapOf() + +/** + * A set containing predefined property types represented as Kotlin type objects that can be used as + * properties in [schemaProperties]. + */ +val propertyTypes = + setOf( + String::class.createType(), + Int::class.createType(), + Long::class.createType(), + Boolean::class.createType(), + Name::class.createType(), + Uuid::class.createType(), + Granularity::class.createType(), + DependenceType::class.createType(), + ) + +internal val log = LoggerFactory.getLogger("Persistence") + +/** + * Returns the [Persistable]'s properties. This DOES NOT include relationships, but only properties + * directly attached to the node/edge. + */ +fun Persistable.properties(): Map { + val properties = mutableMapOf() + for (entry in this::class.schemaProperties) { + val value = entry.value.call(this) + + if (value == null) { + continue + } + + value.convert(entry.key, properties) + } + + return properties +} + +/** + * Runs any conversions that are necessary by [CompositeAttributeConverter] and + * [org.neo4j.ogm.typeconversion.AttributeConverter]. Since both of these classes are Neo4J OGM + * classes, we need to find new base types at some point. + */ +fun Any.convert(originalKey: String, properties: MutableMap) { + // TODO: generalize conversions + if (this is Name && originalKey == "name") { + properties += NameConverter().toGraphProperties(this) + } else if (this is Name) { + properties.put(originalKey, SimpleNameConverter().toGraphProperty(this)) + } else if (this is Granularity) { + properties += DataflowGranularityConverter().toGraphProperties(this) + } else if (this is Enum<*>) { + properties.put(originalKey, this.name) + } else if (this is Uuid) { + properties.put(originalKey, this.toString()) + } else { + properties.put(originalKey, this) + } +} + +/** + * Represents a computed property for obtaining a set of labels associated with a Kotlin class. + * + * Recursively collects labels from the class hierarchy, including superclass labels, and adds the + * simple name of the current class to the set of labels. + * + * Interfaces and the Kotlin base class `Any` are excluded from the labels. The results are cached + * to improve performance. + */ +val KClass<*>.labels: Set + get() { + // Ignore interfaces and the Kotlin base class + if (this.java.isInterface || this == Any::class) { + return setOf() + } + + val cacheKey = this + + // Note: we cannot use computeIfAbsent here, because we are calling our function + // recursively and this would result in a ConcurrentModificationException + if (labelCache.containsKey(cacheKey)) { + return labelCache[cacheKey] ?: setOf() + } + + val labels = mutableSetOf() + labels.addAll(this.superclasses.flatMap { it.labels }) + this.simpleName?.let { labels.add(it) } + + // update the cache + labelCache[cacheKey] = labels + return labels + } + +/** + * Retrieves a map of schema properties (not relationships!) for the given class implementing + * [Persistable]. + * + * This property computes a map that associates property names (as strings) to their corresponding + * [KProperty1] objects, which represent the properties defined in the class. Only properties whose + * return types are included in a predefined set of supported property types ([propertyTypes]) are + * included in the map. + * + * The computed map is cached to optimize subsequent lookups for properties of the same class. + */ +val KClass.schemaProperties: Map> + get() { + // Check, if we already computed the labels for this node's class + return schemaPropertiesCache.computeIfAbsent(this) { + val schema = mutableMapOf>() + val properties = it.memberProperties + for (property in properties) { + if (property.returnType.withNullability(false) in propertyTypes) { + schema.put(property.name, property) + } + } + schema + } + } diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Persistence.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Neo4J.kt similarity index 61% rename from cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Persistence.kt rename to cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Neo4J.kt index fd264b40ceb..b6df3e74784 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Persistence.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Neo4J.kt @@ -26,48 +26,57 @@ package de.fraunhofer.aisec.cpg.persistence import de.fraunhofer.aisec.cpg.TranslationResult -import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.Persistable import de.fraunhofer.aisec.cpg.graph.edges.Edge import de.fraunhofer.aisec.cpg.graph.edges.allEdges -import de.fraunhofer.aisec.cpg.graph.edges.flows.DependenceType -import de.fraunhofer.aisec.cpg.graph.edges.flows.Granularity import de.fraunhofer.aisec.cpg.graph.nodes import de.fraunhofer.aisec.cpg.graph.types.FunctionPointerType import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.SecondOrderType import de.fraunhofer.aisec.cpg.helpers.Benchmark -import de.fraunhofer.aisec.cpg.helpers.neo4j.DataflowGranularityConverter -import de.fraunhofer.aisec.cpg.helpers.neo4j.NameConverter -import de.fraunhofer.aisec.cpg.helpers.neo4j.SimpleNameConverter import de.fraunhofer.aisec.cpg.helpers.toIdentitySet -import kotlin.reflect.KClass -import kotlin.reflect.KProperty1 -import kotlin.reflect.full.createType -import kotlin.reflect.full.memberProperties -import kotlin.reflect.full.superclasses -import kotlin.reflect.full.withNullability -import kotlin.uuid.Uuid import org.neo4j.driver.Session -import org.neo4j.ogm.typeconversion.CompositeAttributeConverter -import org.slf4j.LoggerFactory /** - * docker run \ --name neo4j-apoc \ -p 7474:7474 -p 7687:7687 \ -d \ -e NEO4J_AUTH=neo4j/password \ - * -e NEO4JLABS_PLUGINS='["apoc"]' \ neo4j:5 + * Defines the number of edges to be processed in a single batch operation during persistence. + * + * This constant is used for chunking collections of edges into smaller groups to optimize write + * performance and reduce memory usage when interacting with the Neo4j database. Specifically, it + * determines the maximum size of each chunk of edges to be persisted in one batch operation. */ -val labelCache: MutableMap, Set> = mutableMapOf() - -val schemaPropertiesCache: - MutableMap, Map>> = - mutableMapOf() +const val edgeChunkSize = 10000 -val log = LoggerFactory.getLogger("Persistence") - -val edgeChunkSize = 10000 -val nodeChunkSize = 10000 +/** + * Specifies the maximum number of nodes to be processed in a single chunk during persistence + * operations. + * + * This constant is used to control the size of batches when persisting a list of nodes to the + * database. Breaking the list into chunks of this size helps improve performance and memory + * efficiency during database writes. Each chunk is handled individually, ensuring that operations + * remain manageable even for large data sets. + */ +const val nodeChunkSize = 10000 +/** + * Persists the current [TranslationResult] into a graph database. + * + * This method performs the following actions: + * - Logs information about the number and categories of nodes (e.g., AST nodes, scopes, types, + * languages) and edges that are being persisted. + * - Collects nodes that include AST nodes, scopes, types, and languages, as well as all associated + * edges. + * - Persists the collected nodes and edges. + * - Persists additional relationships between nodes, such as those related to types, scopes, and + * languages. + * - Utilizes a benchmarking mechanism to measure and log the time taken to complete the persistence + * operation. + * + * This method relies on the following context and properties: + * - The [TranslationResult.finalCtx] property for accessing the scope manager, type manager, and + * configuration. + * - A [Session] context to perform persistence actions. + */ context(Session) fun TranslationResult.persist() { val b = Benchmark(Persistable::class.java, "Persisting translation result") @@ -100,6 +109,20 @@ fun TranslationResult.persist() { b.stop() } +/** + * Persists a list of nodes into a database in chunks for efficient processing. + * + * This function utilizes the surrounding [Session] context to execute the database write + * operations. Nodes are processed in chunks of size determined by [nodeChunkSize], and each chunk + * is persisted using Cypher queries. The process is benchmarked using the [Benchmark] utility. + * + * The function generates a list of properties for the nodes, which includes their labels and other + * properties. These properties are used to construct Cypher queries that create nodes in the + * database with the given labels and properties. + * + * The function uses the APOC library for creating nodes in the database. For each node in the list, + * it extracts the labels and properties and executes the Cypher query to persist the node. + */ context(Session) private fun List.persist() { this.chunked(nodeChunkSize).map { chunk -> @@ -122,6 +145,24 @@ private fun List.persist() { } } +/** + * Persists a collection of edges into a Neo4j graph database within the context of a [Session]. + * + * This method ensures that the required index for node IDs is created before proceeding with + * relationship creation. The edges are subdivided into chunks, and for each chunk, the + * relationships are created in the database. Neo4j does not support multiple labels on edges, so + * each edge is duplicated for each assigned label. The created relationships are associated with + * their respective nodes and additional properties derived from the edges. + * + * Constraints: + * - The session context is required to execute write transactions. + * - Edges should define their labels and properties for appropriate persistence. + * + * Mechanisms: + * - An index for [Node] IDs is created (if not already existing) to optimize matching operations. + * - Edges are chunked to avoid overloading transactional operations. + * - Relationship properties and labels are mapped before using database utilities for creation. + */ context(Session) private fun Collection>.persist() { // Create an index for the "id" field of node, because we are "MATCH"ing on it in the edge @@ -193,6 +234,14 @@ private fun List.persistExtraRelationships() { relationships.chunked(10000).map { chunk -> this@Session.createRelationships(chunk) } } +/** + * Creates relationships in a graph database based on provided properties. + * + * @param props A list of maps, where each map represents properties of a relationship including + * keys such as `startId`, `endId`, and `type`. The `startId` and `endId` identify the nodes to + * connect, while `type` defines the type of the relationship. Additional properties for the + * relationship can also be included in the map. + */ private fun Session.createRelationships( props: List>, ) { @@ -215,95 +264,3 @@ private fun Session.createRelationships( } b.stop() } - -/** - * Returns the [Persistable]'s properties. This DOES NOT include relationships, but only properties - * directly attached to the node/edge. - */ -fun Persistable.properties(): Map { - val properties = mutableMapOf() - for (entry in this::class.schemaProperties) { - val value = entry.value.call(this) - - if (value == null) { - continue - } - - value.convert(entry.key, properties) - } - - return properties -} - -/** - * Runs any conversions that are necessary by [CompositeAttributeConverter] and - * [org.neo4j.ogm.typeconversion.AttributeConverter]. Since both of these classes are Neo4J OGM - * classes, we need to find new base types at some point. - */ -fun Any.convert(originalKey: String, properties: MutableMap) { - // TODO: generalize conversions - if (this is Name && originalKey == "name") { - properties += NameConverter().toGraphProperties(this) - } else if (this is Name) { - properties.put(originalKey, SimpleNameConverter().toGraphProperty(this)) - } else if (this is Granularity) { - properties += DataflowGranularityConverter().toGraphProperties(this) - } else if (this is Enum<*>) { - properties.put(originalKey, this.name) - } else if (this is Uuid) { - properties.put(originalKey, this.toString()) - } else { - properties.put(originalKey, this) - } -} - -val KClass<*>.labels: Set - get() { - // Ignore interfaces and the Kotlin base class - if (this.java.isInterface || this == Any::class) { - return setOf() - } - - val cacheKey = this - - // Note: we cannot use computeIfAbsent here, because we are calling our function - // recursively and this would result in a ConcurrentModificationException - if (labelCache.containsKey(cacheKey)) { - return labelCache[cacheKey] ?: setOf() - } - - val labels = mutableSetOf() - labels.addAll(this.superclasses.flatMap { it.labels }) - this.simpleName?.let { labels.add(it) } - - // update the cache - labelCache[cacheKey] = labels - return labels - } - -val propertyTypes = - setOf( - String::class.createType(), - Int::class.createType(), - Long::class.createType(), - Boolean::class.createType(), - Name::class.createType(), - Uuid::class.createType(), - Granularity::class.createType(), - DependenceType::class.createType(), - ) - -val KClass.schemaProperties: Map> - get() { - // Check, if we already computed the labels for this node's class - return schemaPropertiesCache.computeIfAbsent(this) { - val schema = mutableMapOf>() - val properties = it.memberProperties - for (property in properties) { - if (property.returnType.withNullability(false) in propertyTypes) { - schema.put(property.name, property) - } - } - schema - } - } diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt index b286dfc02a5..bacea40c1b1 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt @@ -50,14 +50,10 @@ import picocli.CommandLine import picocli.CommandLine.ArgGroup private const val S_TO_MS_FACTOR = 1000 -private const val TIME_BETWEEN_CONNECTION_TRIES: Long = 2000 -private const val MAX_COUNT_OF_FAILS = 10 private const val EXIT_SUCCESS = 0 private const val EXIT_FAILURE = 1 -private const val VERIFY_CONNECTION = true private const val DEBUG_PARSER = true -private const val AUTO_INDEX = "none" -private const val PROTOCOL = "bolt://" +private const val PROTOCOL = "neo4j://" private const val DEFAULT_HOST = "localhost" private const val DEFAULT_PORT = 7687 @@ -81,6 +77,14 @@ data class JsonGraph(val nodes: List, val edges: List) /** * An application to export the cpg to a neo4j database. + * + * Please make sure, that the [APOC](https://neo4j.com/labs/apoc/) plugin is enabled on your neo4j + * server. It is used in mass-creating nodes and relationships. + * + * For example using docker: + * ``` + * docker run -p 7474:7474 -p 7687:7687 -d -e NEO4J_AUTH=neo4j/password -e NEO4JLABS_PLUGINS='["apoc"]' neo4j:5 + * ``` */ class Application : Callable { From c9376ff63bebeb590d119b5dc834c798a1e166c6 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 11 Dec 2024 16:00:40 +0100 Subject: [PATCH 15/25] Exposed exclusionPatterns as option --- .../de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt index bacea40c1b1..a1ef85e5b6c 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt @@ -252,6 +252,15 @@ class Application : Callable { ) private var topLevel: File? = null + @CommandLine.Option( + names = ["--exclusion-patterns"], + description = + [ + "Set top level directory of project structure. Default: Largest common path of all source files" + ] + ) + private var exclusionPatterns: List = listOf() + @CommandLine.Option( names = ["--benchmark-json"], description = ["Save benchmark results to json file"] @@ -450,6 +459,7 @@ class Application : Callable { .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.jvm.JVMLanguage") .optionalLanguage("de.fraunhofer.aisec.cpg.frontends.ini.IniFileLanguage") .loadIncludes(loadIncludes) + .exclusionPatterns(*exclusionPatterns.toTypedArray()) .addIncludesToGraph(loadIncludes) .debugParser(DEBUG_PARSER) .useUnityBuild(useUnityBuild) From d37453cdf1c0915442c5668815ba6e2d5d5278bd Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 11 Dec 2024 16:49:49 +0100 Subject: [PATCH 16/25] More inclusive way for properties, this also includes location and literal value now --- .../fraunhofer/aisec/cpg/TranslationResult.kt | 2 +- .../aisec/cpg/persistence/Common.kt | 58 ++++++++++++++++++- 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt index fb0e3907cf0..e549c89953f 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt @@ -35,9 +35,9 @@ import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.helpers.MeasurementHolder import de.fraunhofer.aisec.cpg.helpers.StatisticsHolder import de.fraunhofer.aisec.cpg.passes.Pass +import org.neo4j.ogm.annotation.Relationship import java.util.* import java.util.concurrent.ConcurrentHashMap -import org.neo4j.ogm.annotation.Relationship /** * The global (intermediate) result of the translation. A [LanguageFrontend] will initially populate diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt index df0f2e29f33..13ce2cc1bfb 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt @@ -25,19 +25,29 @@ */ package de.fraunhofer.aisec.cpg.persistence +import de.fraunhofer.aisec.cpg.TranslationConfiguration +import de.fraunhofer.aisec.cpg.TranslationContext import de.fraunhofer.aisec.cpg.graph.Name +import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.Persistable import de.fraunhofer.aisec.cpg.graph.edges.flows.DependenceType import de.fraunhofer.aisec.cpg.graph.edges.flows.Granularity +import de.fraunhofer.aisec.cpg.helpers.BenchmarkResults import de.fraunhofer.aisec.cpg.helpers.neo4j.DataflowGranularityConverter +import de.fraunhofer.aisec.cpg.helpers.neo4j.LocationConverter import de.fraunhofer.aisec.cpg.helpers.neo4j.NameConverter import de.fraunhofer.aisec.cpg.helpers.neo4j.SimpleNameConverter +import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation import kotlin.reflect.KClass import kotlin.reflect.KProperty1 +import kotlin.reflect.KTypeProjection +import kotlin.reflect.KVisibility import kotlin.reflect.full.createType +import kotlin.reflect.full.isSubtypeOf import kotlin.reflect.full.memberProperties import kotlin.reflect.full.superclasses import kotlin.reflect.full.withNullability +import kotlin.reflect.jvm.javaType import kotlin.uuid.Uuid import org.neo4j.ogm.typeconversion.CompositeAttributeConverter import org.slf4j.LoggerFactory @@ -116,6 +126,8 @@ fun Any.convert(originalKey: String, properties: MutableMap) { properties += NameConverter().toGraphProperties(this) } else if (this is Name) { properties.put(originalKey, SimpleNameConverter().toGraphProperty(this)) + } else if (this is PhysicalLocation) { + properties += LocationConverter().toGraphProperties(this) } else if (this is Granularity) { properties += DataflowGranularityConverter().toGraphProperties(this) } else if (this is Enum<*>) { @@ -160,6 +172,19 @@ val KClass<*>.labels: Set return labels } +/** A list of specific types that are intended to be ignored for persistence. */ +internal val ignoredTypes = + listOf( + TranslationContext::class.createType(), + TranslationConfiguration::class.createType(), + BenchmarkResults::class.createType(), + KClass::class.createType(listOf(KTypeProjection.STAR)), + ) + +internal val nodeType = Node::class.createType() +internal val collectionType = Collection::class.createType(listOf(KTypeProjection.STAR)) +internal val mapType = Map::class.createType(listOf(KTypeProjection.STAR, KTypeProjection.STAR)) + /** * Retrieves a map of schema properties (not relationships!) for the given class implementing * [Persistable]. @@ -173,15 +198,44 @@ val KClass<*>.labels: Set */ val KClass.schemaProperties: Map> get() { - // Check, if we already computed the labels for this node's class + // Check, if we already computed the properties for this node's class return schemaPropertiesCache.computeIfAbsent(this) { val schema = mutableMapOf>() val properties = it.memberProperties for (property in properties) { - if (property.returnType.withNullability(false) in propertyTypes) { + if (isSimpleProperty(property)) { schema.put(property.name, property) } } schema } } + +/** + * Evaluates whether a given property qualifies as a "simple" property based on its characteristics. + * + * This evaluates to false, when + * - The property is a list (see [collectionType]) + * - The property is a map (see [mapType]) + * - The property is one of our [ignoredTypes] + * - The property is referring to a [Node] + * - The property is an interface + * + * @param property the property to be evaluated, belonging to a class implementing the Persistable + * interface + * @return true if the property satisfies the conditions of being a "simple" property, false + * otherwise + */ +private fun isSimpleProperty(property: KProperty1): Boolean { + val returnType = property.returnType.withNullability(false) + + return when { + property.visibility == KVisibility.PRIVATE -> false + ignoredTypes.any { returnType.isSubtypeOf(it) } -> false + returnType.isSubtypeOf(collectionType) -> false + returnType.isSubtypeOf(mapType) -> false + returnType.withNullability(false).isSubtypeOf(nodeType) -> false + (returnType.javaType as? Class<*>)?.isInterface == true -> false + else -> true + } +} From 3c38788647ed9bd342ddace04a6b1ab9a3d4cc1b Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 11 Dec 2024 16:51:41 +0100 Subject: [PATCH 17/25] Added big integer --- .../main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt | 2 +- .../main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt index e549c89953f..fb0e3907cf0 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt @@ -35,9 +35,9 @@ import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.helpers.MeasurementHolder import de.fraunhofer.aisec.cpg.helpers.StatisticsHolder import de.fraunhofer.aisec.cpg.passes.Pass -import org.neo4j.ogm.annotation.Relationship import java.util.* import java.util.concurrent.ConcurrentHashMap +import org.neo4j.ogm.annotation.Relationship /** * The global (intermediate) result of the translation. A [LanguageFrontend] will initially populate diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt index 13ce2cc1bfb..945680f2729 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt @@ -38,6 +38,7 @@ import de.fraunhofer.aisec.cpg.helpers.neo4j.LocationConverter import de.fraunhofer.aisec.cpg.helpers.neo4j.NameConverter import de.fraunhofer.aisec.cpg.helpers.neo4j.SimpleNameConverter import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation +import java.math.BigInteger import kotlin.reflect.KClass import kotlin.reflect.KProperty1 import kotlin.reflect.KTypeProjection @@ -134,6 +135,8 @@ fun Any.convert(originalKey: String, properties: MutableMap) { properties.put(originalKey, this.name) } else if (this is Uuid) { properties.put(originalKey, this.toString()) + } else if (this is BigInteger) { + properties.put(originalKey, this.toString()) } else { properties.put(originalKey, this) } From ac482aafa71571e91b32958bcdf69c5c2e7dc494 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Fri, 13 Dec 2024 23:02:15 +0100 Subject: [PATCH 18/25] Trying to persist all extra relationships --- .../de/fraunhofer/aisec/cpg/graph/Node.kt | 7 +- .../cpg/graph/declarations/Declaration.kt | 2 + .../graph/declarations/FunctionDeclaration.kt | 16 +- .../declarations/NamespaceDeclaration.kt | 2 + .../graph/declarations/RecordDeclaration.kt | 3 + .../graph/declarations/TemplateDeclaration.kt | 2 + .../TranslationUnitDeclaration.kt | 2 + .../aisec/cpg/graph/statements/CatchClause.kt | 3 + .../aisec/cpg/persistence/Common.kt | 105 ++++++++++- .../aisec/cpg/persistence/DoNotPersist.kt | 29 +++ .../aisec/cpg/persistence/TestCommon.kt | 82 +++++++++ .../fraunhofer/aisec/cpg/persistence/Neo4J.kt | 171 +++++++++--------- 12 files changed, 316 insertions(+), 108 deletions(-) rename {cpg-neo4j => cpg-core}/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt (68%) create mode 100644 cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/DoNotPersist.kt create mode 100644 cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/persistence/TestCommon.kt diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt index 948be849757..d71400fbefc 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt @@ -45,6 +45,7 @@ import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker import de.fraunhofer.aisec.cpg.helpers.neo4j.LocationConverter import de.fraunhofer.aisec.cpg.helpers.neo4j.NameConverter import de.fraunhofer.aisec.cpg.passes.* +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import de.fraunhofer.aisec.cpg.processing.IVisitable import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation import java.util.* @@ -167,7 +168,7 @@ abstract class Node : var astChildren: List = listOf() get() = SubgraphWalker.getAstChildren(this) - @Transient var astParent: Node? = null + @DoNotPersist @Transient var astParent: Node? = null /** Virtual property for accessing [prevEOGEdges] without property edges. */ @PopulatedByPass(EvaluationOrderGraphPass::class) var prevEOG by unwrapping(Node::prevEOGEdges) @@ -190,6 +191,7 @@ abstract class Node : * Virtual property for accessing [nextDFGEdges] that have a * [de.fraunhofer.aisec.cpg.graph.edges.flows.FullDataflowGranularity]. */ + @DoNotPersist @PopulatedByPass(DFGPass::class, ControlFlowSensitiveDFGPass::class) val prevFullDFG: List get() { @@ -213,6 +215,7 @@ abstract class Node : * Virtual property for accessing [nextDFGEdges] that have a * [de.fraunhofer.aisec.cpg.graph.edges.flows.FullDataflowGranularity]. */ + @DoNotPersist @PopulatedByPass(DFGPass::class, ControlFlowSensitiveDFGPass::class) val nextFullDFG: List get() { @@ -253,7 +256,7 @@ abstract class Node : var isImplicit = false /** Required field for object graph mapping. It contains the node id. */ - @Id @GeneratedValue var legacyId: Long? = null + @DoNotPersist @Id @GeneratedValue var legacyId: Long? = null /** Will replace [legacyId] */ var id: Uuid = Uuid.random() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/Declaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/Declaration.kt index fc37bbca33f..76d597190e2 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/Declaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/Declaration.kt @@ -27,6 +27,7 @@ package de.fraunhofer.aisec.cpg.graph.declarations import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.scopes.Symbol +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import org.neo4j.ogm.annotation.NodeEntity /** @@ -40,6 +41,7 @@ import org.neo4j.ogm.annotation.NodeEntity */ @NodeEntity abstract class Declaration : Node() { + @DoNotPersist val symbol: Symbol get() { return this.name.localName diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt index b6072808118..553b4f541fd 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt @@ -34,6 +34,7 @@ import de.fraunhofer.aisec.cpg.graph.statements.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import de.fraunhofer.aisec.cpg.graph.types.Type +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import java.util.* import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.Relationship @@ -41,7 +42,7 @@ import org.neo4j.ogm.annotation.Relationship /** Represents the declaration or definition of a function. */ open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder, EOGStarterHolder { @Relationship("BODY") var bodyEdge = astOptionalEdgeOf(label = "BODY") - /** The function body. Usually a [Block]. */ + /** The function body. Usualfly a [Block]. */ var body by unwrapping(FunctionDeclaration::bodyEdge) /** The list of function parameters. */ @@ -134,16 +135,6 @@ open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder, EOGStart return parameters.map { it.default } } - val defaultParameterSignature: List // TODO: What's this property? - get() = - parameters.map { - if (it.default != null) { - it.type - } else { - unknownType() - } - } - val signatureTypes: List get() = parameters.map { it.type } @@ -154,6 +145,7 @@ open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder, EOGStart .toString() } + @DoNotPersist override val eogStarters: List get() = listOfNotNull(this) @@ -179,11 +171,11 @@ open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder, EOGStart } } + @DoNotPersist override val declarations: List get() { val list = ArrayList() list.addAll(parameters) - list.addAll(records) return list } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt index 6337a682a0b..f90d845ed22 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt @@ -29,6 +29,7 @@ import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.Statement +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import java.util.Objects import org.neo4j.ogm.annotation.Relationship @@ -75,6 +76,7 @@ class NamespaceDeclaration : Declaration(), DeclarationHolder, StatementHolder, override var statements by unwrapping(NamespaceDeclaration::statementEdges) + @DoNotPersist override val eogStarters: List get() { val list = mutableListOf() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt index 27f5fa01b7b..b7cdeeb583c 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt @@ -33,6 +33,7 @@ import de.fraunhofer.aisec.cpg.graph.statements.Statement import de.fraunhofer.aisec.cpg.graph.types.DeclaresType import de.fraunhofer.aisec.cpg.graph.types.ObjectType import de.fraunhofer.aisec.cpg.graph.types.Type +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.Relationship import org.neo4j.ogm.annotation.Transient @@ -120,6 +121,7 @@ open class RecordDeclaration : templateEdges.removeIf { it.end == templateDeclaration } } + @DoNotPersist override val declarations: List get() { val list = ArrayList() @@ -162,6 +164,7 @@ open class RecordDeclaration : .toString() } + @DoNotPersist override val eogStarters: List get() { val list = mutableListOf() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TemplateDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TemplateDeclaration.kt index 4048a04da41..d960b6e02c1 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TemplateDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TemplateDeclaration.kt @@ -30,6 +30,7 @@ import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.edges.Edge.Companion.propertyEqualsList import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf import de.fraunhofer.aisec.cpg.graph.edges.unwrapping +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import org.neo4j.ogm.annotation.NodeEntity import org.neo4j.ogm.annotation.Relationship @@ -81,6 +82,7 @@ abstract class TemplateDeclaration : Declaration(), DeclarationHolder { return defaults } + @DoNotPersist override val declarations: List get() { val list = ArrayList() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt index afd80853394..df2611f2f89 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt @@ -30,6 +30,7 @@ import de.fraunhofer.aisec.cpg.graph.edges.Edge.Companion.propertyEqualsList import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.Statement +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import java.util.* import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.Relationship @@ -74,6 +75,7 @@ class TranslationUnitDeclaration : .toString() } + @DoNotPersist override val eogStarters: List get() { val list = mutableListOf() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CatchClause.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CatchClause.kt index 025de94b67a..ba52c16d691 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CatchClause.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CatchClause.kt @@ -32,6 +32,7 @@ import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.edges.ast.astOptionalEdgeOf import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import java.util.Objects import org.neo4j.ogm.annotation.Relationship @@ -44,9 +45,11 @@ class CatchClause : Statement(), BranchingNode, EOGStarterHolder { @Relationship(value = "BODY") var bodyEdge = astOptionalEdgeOf(label = "BODY") var body by unwrapping(CatchClause::bodyEdge) + @DoNotPersist override val branchedBy: Node? get() = parameter + @DoNotPersist override val eogStarters: List get() = listOf(this) diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt similarity index 68% rename from cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt rename to cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt index 945680f2729..d2d9a64e2ae 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt @@ -30,6 +30,8 @@ import de.fraunhofer.aisec.cpg.TranslationContext import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.Persistable +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeCollection +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeList import de.fraunhofer.aisec.cpg.graph.edges.flows.DependenceType import de.fraunhofer.aisec.cpg.graph.edges.flows.Granularity import de.fraunhofer.aisec.cpg.helpers.BenchmarkResults @@ -42,16 +44,20 @@ import java.math.BigInteger import kotlin.reflect.KClass import kotlin.reflect.KProperty1 import kotlin.reflect.KTypeProjection +import kotlin.reflect.KVariance import kotlin.reflect.KVisibility import kotlin.reflect.full.createType +import kotlin.reflect.full.hasAnnotation import kotlin.reflect.full.isSubtypeOf import kotlin.reflect.full.memberProperties import kotlin.reflect.full.superclasses import kotlin.reflect.full.withNullability +import kotlin.reflect.jvm.javaField import kotlin.reflect.jvm.javaType import kotlin.uuid.Uuid +import org.neo4j.ogm.annotation.Relationship +import org.neo4j.ogm.annotation.Relationship.Direction.INCOMING import org.neo4j.ogm.typeconversion.CompositeAttributeConverter -import org.slf4j.LoggerFactory /** * A cache used to store and retrieve sets of labels associated with specific Kotlin class types. @@ -79,6 +85,11 @@ val schemaPropertiesCache: MutableMap, Map>> = mutableMapOf() +/** A cache mapping classes of type [Persistable] to their respective properties. */ +val schemaRelationshipCache: + MutableMap, Map>> = + mutableMapOf() + /** * A set containing predefined property types represented as Kotlin type objects that can be used as * properties in [schemaProperties]. @@ -95,8 +106,6 @@ val propertyTypes = DependenceType::class.createType(), ) -internal val log = LoggerFactory.getLogger("Persistence") - /** * Returns the [Persistable]'s properties. This DOES NOT include relationships, but only properties * directly attached to the node/edge. @@ -186,6 +195,10 @@ internal val ignoredTypes = internal val nodeType = Node::class.createType() internal val collectionType = Collection::class.createType(listOf(KTypeProjection.STAR)) +internal val collectionOfNodeType = + Collection::class.createType(listOf(KTypeProjection(variance = KVariance.OUT, type = nodeType))) +internal val edgeCollectionType = + EdgeCollection::class.createType(listOf(KTypeProjection.STAR, KTypeProjection.STAR)) internal val mapType = Map::class.createType(listOf(KTypeProjection.STAR, KTypeProjection.STAR)) /** @@ -214,15 +227,32 @@ val KClass.schemaProperties: Map.schemaRelationships: Map> + get() { + // Check, if we already computed the relationship for this node's class + return schemaRelationshipCache.computeIfAbsent(this) { + val schema = mutableMapOf>() + val properties = it.memberProperties + for (property in properties) { + if (isRelationship(property)) { + val name = property.relationshipName + schema.put(name, property) + } + } + schema + } + } + /** * Evaluates whether a given property qualifies as a "simple" property based on its characteristics. * - * This evaluates to false, when - * - The property is a list (see [collectionType]) - * - The property is a map (see [mapType]) - * - The property is one of our [ignoredTypes] - * - The property is referring to a [Node] - * - The property is an interface + * This evaluates to true, when + * - The property is not a list (see [collectionType]) + * - The property is not a map (see [mapType]) + * - The property is not one of our [ignoredTypes] + * - The property is not referring to a [Node] + * - The property is not an interface + * - The property does not have the annotation [DoNotPersist] * * @param property the property to be evaluated, belonging to a class implementing the Persistable * interface @@ -233,12 +263,67 @@ private fun isSimpleProperty(property: KProperty1): Boolean val returnType = property.returnType.withNullability(false) return when { + property.hasAnnotation() -> false property.visibility == KVisibility.PRIVATE -> false ignoredTypes.any { returnType.isSubtypeOf(it) } -> false returnType.isSubtypeOf(collectionType) -> false returnType.isSubtypeOf(mapType) -> false - returnType.withNullability(false).isSubtypeOf(nodeType) -> false + returnType.isSubtypeOf(nodeType) -> false (returnType.javaType as? Class<*>)?.isInterface == true -> false else -> true } } + +/** + * Evaluates whether a given property qualifies as a "relationship" based on its characteristics. + * + * This evaluates to true, when + * - The property is not a delegate (Note: this might change in the future, once we re-design node + * properties if Neo4J OGM is completely removed) + * - The property is an [EdgeList] + * - The property is referring to a [Collection] of [Node] objects + * - The property is referring to a [Node] + * - The property does not have the annotation [DoNotPersist] + * - The property does not have the annotation [org.neo4j.ogm.annotation.Relationship] with an + * incoming direction (Note: We will replace this with our own annotation at some point) + * + * @param property the property to be evaluated, belonging to a class implementing the Persistable + * interface + * @return true if the property satisfies the conditions of being a "relationship", false otherwise + */ +private fun isRelationship(property: KProperty1): Boolean { + val returnType = property.returnType.withNullability(false) + + return when { + property.hasAnnotation() -> false + property.javaField?.type?.simpleName?.contains("Delegate") == true -> false + property.javaField?.getAnnotation(Relationship::class.java)?.direction == INCOMING -> false + property.visibility == KVisibility.PRIVATE -> false + returnType.isSubtypeOf(edgeCollectionType) -> true + returnType.isSubtypeOf(collectionOfNodeType) -> true + returnType.isSubtypeOf(nodeType) -> true + else -> false + } +} + +val KProperty1.relationshipName: String + get() { + // If we have a (legacy) Neo4J annotation for our relationship, we take this one + // Note: We will replace this with something else in the future + val value = this.javaField?.getAnnotation(Relationship::class.java)?.value + if (value != null && value != "") { + return value + } + + // If the name ends with "Edge", we cut that of, since we always have two + // variables for real edges, one called "exampleEdge" and one just called + // "example" (which is only a delegate), but the name we want is "example". + // + // Replace camel case with UPPER_CASE + return this.name.substringBeforeLast("Edge").toUpperSnakeCase() + } + +fun String.toUpperSnakeCase(): String { + val pattern = "(?<=.)[A-Z]".toRegex() + return this.replace(pattern, "_$0").uppercase() +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/DoNotPersist.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/DoNotPersist.kt new file mode 100644 index 00000000000..3843d71344d --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/DoNotPersist.kt @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.persistence + +/** This annotation is used to denote that this property or class should not be persisted */ +@Target(AnnotationTarget.PROPERTY, AnnotationTarget.CLASS) annotation class DoNotPersist() diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/persistence/TestCommon.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/persistence/TestCommon.kt new file mode 100644 index 00000000000..c53fc609a10 --- /dev/null +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/persistence/TestCommon.kt @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.persistence + +import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration +import kotlin.test.Test +import kotlin.test.assertEquals + +class TestCommon { + @Test + fun testSchemaProperties() { + val properties = FunctionDeclaration::class.schemaProperties + assertEquals( + setOf( + "complexity", + "isDefinition", + "signature", + "argumentIndex", + "code", + "comment", + "file", + "id", + "isImplicit", + "isInferred", + "location", + "name" + ), + properties.keys + ) + } + + @Test + fun testSchemaRelationships() { + val relationships = FunctionDeclaration::class.schemaRelationships + assertEquals( + listOf( + "ANNOTATIONS", + "ASSIGNED_TYPES", + "AST", + "BODY", + "CDG", + "DEFINES", + "DFG", + "EOG", + "LANGUAGE", + "OVERRIDES", + "PARAMETERS", + "PDG", + "RETURN_TYPES", + "SCOPE", + "SIGNATURE_TYPES", + "THROWS_TYPES", + "TYPE", + "USAGE", + ), + relationships.keys.sorted() + ) + } +} diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Neo4J.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Neo4J.kt index b6df3e74784..09f6c2fb674 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Neo4J.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Neo4J.kt @@ -28,15 +28,13 @@ package de.fraunhofer.aisec.cpg.persistence import de.fraunhofer.aisec.cpg.TranslationResult import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.Persistable -import de.fraunhofer.aisec.cpg.graph.edges.Edge -import de.fraunhofer.aisec.cpg.graph.edges.allEdges +import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeCollection import de.fraunhofer.aisec.cpg.graph.nodes -import de.fraunhofer.aisec.cpg.graph.types.FunctionPointerType -import de.fraunhofer.aisec.cpg.graph.types.HasType -import de.fraunhofer.aisec.cpg.graph.types.SecondOrderType import de.fraunhofer.aisec.cpg.helpers.Benchmark -import de.fraunhofer.aisec.cpg.helpers.toIdentitySet +import de.fraunhofer.aisec.cpg.helpers.IdentitySet +import de.fraunhofer.aisec.cpg.helpers.identitySetOf import org.neo4j.driver.Session +import org.slf4j.LoggerFactory /** * Defines the number of edges to be processed in a single batch operation during persistence. @@ -58,6 +56,10 @@ const val edgeChunkSize = 10000 */ const val nodeChunkSize = 10000 +internal val log = LoggerFactory.getLogger("Persistence") + +internal typealias Relationship = Map + /** * Persists the current [TranslationResult] into a graph database. * @@ -82,29 +84,21 @@ fun TranslationResult.persist() { val b = Benchmark(Persistable::class.java, "Persisting translation result") val astNodes = this@persist.nodes - val scopes = this.finalCtx.scopeManager.filterScopes { true } - val languages = this.finalCtx.config.languages - val types = - (this.finalCtx.typeManager.firstOrderTypes + this.finalCtx.typeManager.secondOrderTypes) - .toIdentitySet() - val nodes = listOf(astNodes, scopes, languages, types).flatten() - val edges = this@persist.allEdges>() + val connected = astNodes.flatMap { it.connectedNodes } + val nodes = astNodes + connected log.info( - "Persisting {} nodes: AST nodes ({}), types ({}), scopes ({}) and languages ({})", + "Persisting {} nodes: AST nodes ({}), other nodes ({})", nodes.size, astNodes.size, - types.size, - scopes.size, - languages.size + connected.size ) nodes.persist() - log.info("Persisting {} edges", edges.size) - edges.persist() + val relationships = nodes.collectRelationships() - log.info("Persisting extra relationships (types, scopes, languages)") - nodes.persistExtraRelationships() + log.info("Persisting {} relationships", relationships.size) + relationships.persist() b.stop() } @@ -164,74 +158,14 @@ private fun List.persist() { * - Relationship properties and labels are mapped before using database utilities for creation. */ context(Session) -private fun Collection>.persist() { +private fun Collection.persist() { // Create an index for the "id" field of node, because we are "MATCH"ing on it in the edge // creation. We need to wait for this to be finished this@Session.executeWrite { tx -> tx.run("CREATE INDEX IF NOT EXISTS FOR (n:Node) ON (n.id)").consume() } - this.chunked(edgeChunkSize).map { chunk -> - createRelationships( - chunk.flatMap { edge -> - // Since Neo4J does not support multiple labels on edges, but we do internally, we - // duplicate the edge for each label - edge.labels.map { label -> - mapOf( - "startId" to edge.start.id.toString(), - "endId" to edge.end.id.toString(), - "type" to label - ) + edge.properties() - } - } - ) - } -} - -/** - * Some of our relationships are not real "edges" (i.e., [Edge]) (yet). We need to handle these case - * separately (for now). - */ -context(Session) -private fun List.persistExtraRelationships() { - val relationships = - this.flatMap { - listOfNotNull( - mapOf( - "startId" to it.id.toString(), - "endId" to it.language?.id.toString(), - "type" to "LANGUAGE" - ), - mapOf( - "startId" to it.id.toString(), - "endId" to it.scope?.id.toString(), - "type" to "SCOPE" - ), - if (it is HasType) { - mapOf( - "startId" to it.id.toString(), - "endId" to it.type.id.toString(), - "type" to "TYPE" - ) - } else if (it is SecondOrderType) { - mapOf( - "startId" to it.id.toString(), - "endId" to it.elementType.id.toString(), - "type" to "ELEMENT_TYPE" - ) - } else if (it is FunctionPointerType) { - mapOf( - "startId" to it.id.toString(), - "endId" to it.returnType.id.toString(), - "type" to "RETURN_TYPE" - ) - } else { - null - } - ) - } - - relationships.chunked(10000).map { chunk -> this@Session.createRelationships(chunk) } + this.chunked(edgeChunkSize).map { chunk -> createRelationships(chunk) } } /** @@ -243,7 +177,7 @@ private fun List.persistExtraRelationships() { * relationship can also be included in the map. */ private fun Session.createRelationships( - props: List>, + props: List, ) { val b = Benchmark(Persistable::class.java, "Persisting chunk of ${props.size} relationships") val params = mapOf("props" to props) @@ -264,3 +198,72 @@ private fun Session.createRelationships( } b.stop() } + +/** + * Returns all [Node] objects that are connected with this node with some kind of relationship + * defined in [schemaRelationships]. + */ +val Persistable.connectedNodes: IdentitySet + get() { + val nodes = identitySetOf() + + for (entry in this::class.schemaRelationships) { + val value = entry.value.call(this) + if (value is EdgeCollection<*, *>) { + nodes += value.toNodeCollection() + } else if (value is List<*>) { + nodes += value.filterIsInstance() + } else if (value is Node) { + nodes += value + } + } + + return nodes + } + +private fun List.collectRelationships(): List { + val relationships = mutableListOf() + + for (node in this) { + for (entry in node::class.schemaRelationships) { + val value = entry.value.call(node) + if (value is EdgeCollection<*, *>) { + relationships += + value.map { edge -> + mapOf( + "startId" to edge.start.id.toString(), + "endId" to edge.end.id.toString(), + "type" to entry.key + ) + edge.properties() + } + } else if (value is List<*>) { + relationships += + value.filterIsInstance().map { end -> + mapOf( + "startId" to node.id.toString(), + "endId" to end.id.toString(), + "type" to entry.key + ) + } + } else if (value is Node) { + relationships += + mapOf( + "startId" to node.id.toString(), + "endId" to value.id.toString(), + "type" to entry.key + ) + } + } + } + + // Since Neo4J does not support multiple labels on edges, but we do internally, we + // duplicate the edge for each label + /*edge.labels.map { label -> + mapOf( + "startId" to edge.start.id.toString(), + "endId" to edge.end.id.toString(), + "type" to label + ) + edge.properties() + }*/ + return relationships +} From a7bd3df4fca77fcf32346f18be1060d923a7084b Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Fri, 13 Dec 2024 23:21:35 +0100 Subject: [PATCH 19/25] Revert labels --- .../fraunhofer/aisec/cpg/TranslationResult.kt | 2 +- .../aisec/cpg/frontends/Language.kt | 2 +- .../fraunhofer/aisec/cpg/graph/Component.kt | 2 +- .../de/fraunhofer/aisec/cpg/graph/Node.kt | 2 +- .../declarations/EnumConstantDeclaration.kt | 3 +- .../cpg/graph/declarations/EnumDeclaration.kt | 2 +- .../graph/declarations/FunctionDeclaration.kt | 4 +-- .../FunctionTemplateDeclaration.kt | 2 +- .../graph/declarations/IncludeDeclaration.kt | 4 +-- .../graph/declarations/MethodDeclaration.kt | 3 +- .../declarations/NamespaceDeclaration.kt | 4 +-- .../declarations/ParameterDeclaration.kt | 2 +- .../graph/declarations/RecordDeclaration.kt | 12 ++++---- .../declarations/RecordTemplateDeclaration.kt | 2 +- .../graph/declarations/TemplateDeclaration.kt | 2 +- .../TranslationUnitDeclaration.kt | 8 ++--- .../graph/declarations/TupleDeclaration.kt | 8 ++--- .../declarations/TypeParameterDeclaration.kt | 2 +- .../graph/declarations/VariableDeclaration.kt | 3 +- .../aisec/cpg/graph/edges/ast/AstEdge.kt | 12 +++----- .../cpg/graph/edges/ast/TemplateArgument.kt | 8 ++--- .../cpg/graph/statements/AssertStatement.kt | 6 ++-- .../cpg/graph/statements/CaseStatement.kt | 2 +- .../aisec/cpg/graph/statements/CatchClause.kt | 5 ++-- .../graph/statements/DeclarationStatement.kt | 2 +- .../aisec/cpg/graph/statements/DoStatement.kt | 3 +- .../cpg/graph/statements/ForEachStatement.kt | 5 ++-- .../cpg/graph/statements/ForStatement.kt | 10 +++---- .../aisec/cpg/graph/statements/IfStatement.kt | 13 ++++---- .../cpg/graph/statements/LabelStatement.kt | 5 ++-- .../cpg/graph/statements/LoopStatement.kt | 5 ++-- .../cpg/graph/statements/ReturnStatement.kt | 3 +- .../aisec/cpg/graph/statements/Statement.kt | 2 +- .../cpg/graph/statements/SwitchStatement.kt | 10 +++---- .../graph/statements/SynchronizedStatement.kt | 5 ++-- .../cpg/graph/statements/ThrowExpression.kt | 5 ++-- .../cpg/graph/statements/TryStatement.kt | 13 ++++---- .../cpg/graph/statements/WhileStatement.kt | 5 ++-- .../expressions/AssignExpression.kt | 12 +++----- .../statements/expressions/BinaryOperator.kt | 2 -- .../cpg/graph/statements/expressions/Block.kt | 2 +- .../statements/expressions/CallExpression.kt | 9 +++--- .../statements/expressions/CastExpression.kt | 1 - .../expressions/CollectionComprehension.kt | 4 +-- .../expressions/ComprehensionExpression.kt | 8 ++--- .../expressions/ConditionalExpression.kt | 7 +---- .../expressions/ConstructExpression.kt | 3 +- .../expressions/DeleteExpression.kt | 2 +- .../statements/expressions/ExpressionList.kt | 2 +- .../expressions/InitializerListExpression.kt | 6 ++-- .../expressions/KeyValueExpression.kt | 6 ++-- .../expressions/LambdaExpression.kt | 6 +--- .../expressions/MemberExpression.kt | 1 - .../expressions/NewArrayExpression.kt | 5 ++-- .../statements/expressions/NewExpression.kt | 5 ++-- .../statements/expressions/RangeExpression.kt | 6 ++-- .../expressions/SubscriptExpression.kt | 6 +--- .../statements/expressions/UnaryOperator.kt | 1 - .../fraunhofer/aisec/cpg/persistence/Neo4J.kt | 30 +++++++++++++++++-- 59 files changed, 136 insertions(+), 176 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt index fb0e3907cf0..05ace1a4dc8 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt @@ -54,7 +54,7 @@ class TranslationResult( var finalCtx: TranslationContext, ) : Node(), StatisticsHolder { - @Relationship("COMPONENTS") val componentEdges = astEdgesOf(label = "COMPONENTS") + @Relationship("COMPONENTS") val componentEdges = astEdgesOf() /** * Entry points to the CPG: "SoftwareComponent" refer to programs, application, other "bundles" * of software. diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt index 9503ba8bc8e..6192a7f1308 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt @@ -323,7 +323,7 @@ abstract class Language> : Node() { // matches val source = result.source if (this is HasTemplates && source is CallExpression) { - source.templateArgumentEdges = TemplateArguments(source, label = "TEMPLATE_ARGUMENTS") + source.templateArgumentEdges = TemplateArguments(source) val (ok, candidates) = this.handleTemplateFunctionCalls( null, diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Component.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Component.kt index 73c9d1c47d1..10f4edaf7dd 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Component.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Component.kt @@ -43,7 +43,7 @@ import org.neo4j.ogm.annotation.Transient */ open class Component : Node() { @Relationship("TRANSLATION_UNITS") - val translationUnitEdges = astEdgesOf(label = "TRANSLATION_UNITS") + val translationUnitEdges = astEdgesOf() /** All translation units belonging to this application. */ val translationUnits by unwrapping(Component::translationUnitEdges) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt index d71400fbefc..de06b9d2be0 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt @@ -265,7 +265,7 @@ abstract class Node : var argumentIndex = 0 /** List of annotations associated with that node. */ - @Relationship("ANNOTATIONS") var annotationEdges = astEdgesOf(label = "ANNOTATIONS") + @Relationship("ANNOTATIONS") var annotationEdges = astEdgesOf() var annotations by unwrapping(Node::annotationEdges) /** diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumConstantDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumConstantDeclaration.kt index 7786e8fe234..4ef95234bc6 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumConstantDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumConstantDeclaration.kt @@ -36,7 +36,6 @@ import org.neo4j.ogm.annotation.Relationship * explicit initializer value. */ class EnumConstantDeclaration : ValueDeclaration(), HasInitializer { - @Relationship("INITIALIZER") - var initializerEdge = astOptionalEdgeOf(label = "INITIALIZER") + @Relationship("INITIALIZER") var initializerEdge = astOptionalEdgeOf() override var initializer by unwrapping(EnumConstantDeclaration::initializerEdge) } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumDeclaration.kt index b8815215318..c0e912c32f0 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/EnumDeclaration.kt @@ -32,7 +32,7 @@ import org.neo4j.ogm.annotation.Relationship class EnumDeclaration : RecordDeclaration() { @Relationship(value = "ENTRIES", direction = Relationship.Direction.OUTGOING) - var entryEdges = astEdgesOf(label = "ENTRIES") + var entryEdges = astEdgesOf() var entries by unwrapping(EnumDeclaration::entryEdges) override fun toString(): String { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt index 553b4f541fd..d489bd4c8a5 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt @@ -41,13 +41,13 @@ import org.neo4j.ogm.annotation.Relationship /** Represents the declaration or definition of a function. */ open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder, EOGStarterHolder { - @Relationship("BODY") var bodyEdge = astOptionalEdgeOf(label = "BODY") + @Relationship("BODY") var bodyEdge = astOptionalEdgeOf() /** The function body. Usualfly a [Block]. */ var body by unwrapping(FunctionDeclaration::bodyEdge) /** The list of function parameters. */ @Relationship(value = "PARAMETERS", direction = Relationship.Direction.OUTGOING) - var parameterEdges = astEdgesOf(label = "PARAMETERS") + var parameterEdges = astEdgesOf() /** Virtual property for accessing [parameterEdges] without property edges. */ var parameters by unwrapping(FunctionDeclaration::parameterEdges) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionTemplateDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionTemplateDeclaration.kt index 5c393d18fe0..9b9bcdca326 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionTemplateDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionTemplateDeclaration.kt @@ -39,7 +39,7 @@ class FunctionTemplateDeclaration : TemplateDeclaration() { * expansion pass for each instantiation of the FunctionTemplate there will be a realization */ @Relationship(value = "REALIZATION", direction = Relationship.Direction.OUTGOING) - val realizationEdges = astEdgesOf(label = "REALIZATION") + val realizationEdges = astEdgesOf() val realization by unwrapping(FunctionTemplateDeclaration::realizationEdges) override val realizations: List diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/IncludeDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/IncludeDeclaration.kt index 994b51cbd8c..da8e3edb144 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/IncludeDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/IncludeDeclaration.kt @@ -35,11 +35,11 @@ import org.neo4j.ogm.annotation.Relationship /** This declaration represents either an include or an import, depending on the language. */ class IncludeDeclaration : Declaration() { @Relationship(value = "INCLUDES", direction = Relationship.Direction.OUTGOING) - val includeEdges = astEdgesOf(label = "INCLUDES") + val includeEdges = astEdgesOf() val includes by unwrapping(IncludeDeclaration::includeEdges) @Relationship(value = "PROBLEMS", direction = Relationship.Direction.OUTGOING) - val problemEdges = astEdgesOf(label = "PROBLEMS") + val problemEdges = astEdgesOf() val problems by unwrapping(IncludeDeclaration::problemEdges) /** diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/MethodDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/MethodDeclaration.kt index 45525f33c1c..422e69fcefd 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/MethodDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/MethodDeclaration.kt @@ -43,8 +43,7 @@ open class MethodDeclaration : FunctionDeclaration() { */ open var recordDeclaration: RecordDeclaration? = null - @Relationship("RECEIVER") - var receiverEdge = astOptionalEdgeOf(label = "RECEIVER") + @Relationship("RECEIVER") var receiverEdge = astOptionalEdgeOf() /** * The receiver variable of this method. In most cases, this variable is called `this`, but in * some languages, it is `self` (e.g. in Rust or Python) or can be freely named (e.g. in diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt index f90d845ed22..20bebae19d6 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.kt @@ -49,12 +49,12 @@ class NamespaceDeclaration : Declaration(), DeclarationHolder, StatementHolder, * Edges to nested namespaces, records, functions, fields etc. contained in the current * namespace. */ - val declarationEdges = astEdgesOf(label = "DECLARATIONS") + val declarationEdges = astEdgesOf() override val declarations by unwrapping(NamespaceDeclaration::declarationEdges) /** The list of statements. */ @Relationship(value = "STATEMENTS", direction = Relationship.Direction.OUTGOING) - override var statementEdges = astEdgesOf(label = "STATEMENTS") + override var statementEdges = astEdgesOf() /** * In some languages, there is a relationship between paths / directories and the package diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ParameterDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ParameterDeclaration.kt index 9b0fb81753d..cd820498d05 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ParameterDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/ParameterDeclaration.kt @@ -37,7 +37,7 @@ class ParameterDeclaration : ValueDeclaration(), HasDefault { var isVariadic = false @Relationship(value = "DEFAULT", direction = Relationship.Direction.OUTGOING) - var defaultValueEdge = astOptionalEdgeOf(label = "DEFAULT") + var defaultValueEdge = astOptionalEdgeOf() private var defaultValue by unwrapping(ParameterDeclaration::defaultValueEdge) var modifiers: List = mutableListOf() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt index b7cdeeb583c..23440b37c2c 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt @@ -45,28 +45,28 @@ open class RecordDeclaration : var kind: String? = null @Relationship(value = "FIELDS", direction = Relationship.Direction.OUTGOING) - var fieldEdges = astEdgesOf(label = "FIELDS") + var fieldEdges = astEdgesOf() var fields by unwrapping(RecordDeclaration::fieldEdges) @Relationship(value = "METHODS", direction = Relationship.Direction.OUTGOING) - var methodEdges = astEdgesOf(label = "METHODS") + var methodEdges = astEdgesOf() var methods by unwrapping(RecordDeclaration::methodEdges) @Relationship(value = "CONSTRUCTORS", direction = Relationship.Direction.OUTGOING) - var constructorEdges = astEdgesOf(label = "CONSTRUCTORS") + var constructorEdges = astEdgesOf() var constructors by unwrapping(RecordDeclaration::constructorEdges) @Relationship(value = "RECORDS", direction = Relationship.Direction.OUTGOING) - var recordEdges = astEdgesOf(label = "RECORDS") + var recordEdges = astEdgesOf() var records by unwrapping(RecordDeclaration::recordEdges) @Relationship(value = "TEMPLATES", direction = Relationship.Direction.OUTGOING) - var templateEdges = astEdgesOf(label = "TEMPLATES") + var templateEdges = astEdgesOf() var templates by unwrapping(RecordDeclaration::templateEdges) /** The list of statements. */ @Relationship(value = "STATEMENTS", direction = Relationship.Direction.OUTGOING) - override var statementEdges = astEdgesOf(label = "STATEMENTS") + override var statementEdges = astEdgesOf() override var statements by unwrapping(RecordDeclaration::statementEdges) @Transient var superClasses: MutableList = ArrayList() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordTemplateDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordTemplateDeclaration.kt index 028ef222e24..56e4b316454 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordTemplateDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordTemplateDeclaration.kt @@ -39,7 +39,7 @@ class RecordTemplateDeclaration : TemplateDeclaration() { * expansion pass for each instantiation of the ClassTemplate there will be a realization */ @Relationship(value = "REALIZATION", direction = Relationship.Direction.OUTGOING) - val realizationEdges = astEdgesOf(label = "REALIZATION") + val realizationEdges = astEdgesOf() override val realizations by unwrapping(RecordTemplateDeclaration::realizationEdges) override fun addDeclaration(declaration: Declaration) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TemplateDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TemplateDeclaration.kt index d960b6e02c1..f49c43fda32 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TemplateDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TemplateDeclaration.kt @@ -52,7 +52,7 @@ abstract class TemplateDeclaration : Declaration(), DeclarationHolder { /** Parameters the Template requires for instantiation */ @Relationship(value = "PARAMETERS", direction = Relationship.Direction.OUTGOING) - var parameterEdges = astEdgesOf(label = "PARAMETERS") + var parameterEdges = astEdgesOf() val parameters by unwrapping(TemplateDeclaration::parameterEdges) val parametersWithDefaults: List diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt index df2611f2f89..4b479bcd367 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.kt @@ -40,22 +40,22 @@ class TranslationUnitDeclaration : Declaration(), DeclarationHolder, StatementHolder, EOGStarterHolder { /** A list of declarations within this unit. */ @Relationship(value = "DECLARATIONS", direction = Relationship.Direction.OUTGOING) - val declarationEdges = astEdgesOf(label = "DECLARATIONS") + val declarationEdges = astEdgesOf() override val declarations by unwrapping(TranslationUnitDeclaration::declarationEdges) /** A list of includes within this unit. */ @Relationship(value = "INCLUDES", direction = Relationship.Direction.OUTGOING) - val includeEdges = astEdgesOf(label = "INCLUDES") + val includeEdges = astEdgesOf() val includes by unwrapping(TranslationUnitDeclaration::includeEdges) /** A list of namespaces within this unit. */ @Relationship(value = "NAMESPACES", direction = Relationship.Direction.OUTGOING) - val namespaceEdges = astEdgesOf(label = "NAMESPACES") + val namespaceEdges = astEdgesOf() val namespaces by unwrapping(TranslationUnitDeclaration::namespaceEdges) /** The list of statements. */ @Relationship(value = "STATEMENTS", direction = Relationship.Direction.OUTGOING) - override var statementEdges = astEdgesOf(label = "STATEMENTS") + override var statementEdges = astEdgesOf() override var statements by unwrapping(TranslationUnitDeclaration::statementEdges) override fun addDeclaration(declaration: Declaration) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TupleDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TupleDeclaration.kt index 4f70b631e3b..2755ad7a836 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TupleDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TupleDeclaration.kt @@ -64,11 +64,9 @@ import de.fraunhofer.aisec.cpg.graph.types.TupleType class TupleDeclaration : VariableDeclaration() { /** The list of elements in this tuple. */ var elementEdges = - astEdgesOf( - label = "ELEMENTS", - onAdd = { registerTypeObserver(it.end) }, - onRemove = { unregisterTypeObserver(it.end) } - ) + astEdgesOf(onAdd = { registerTypeObserver(it.end) }) { + unregisterTypeObserver(it.end) + } var elements by unwrapping(TupleDeclaration::elementEdges) override var name: Name diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TypeParameterDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TypeParameterDeclaration.kt index cd558339e99..2f061836a24 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TypeParameterDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TypeParameterDeclaration.kt @@ -35,7 +35,7 @@ import org.neo4j.ogm.annotation.Relationship /** A declaration of a type template parameter */ class TypeParameterDeclaration : ValueDeclaration(), HasDefault { @Relationship(value = "DEFAULT", direction = Relationship.Direction.OUTGOING) - var defaultEdge = astOptionalEdgeOf(label = "DEFAULT") + var defaultEdge = astOptionalEdgeOf() /** TemplateParameters can define a default for the type parameter. */ override var default by unwrapping(TypeParameterDeclaration::defaultEdge) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt index cbaa05e56d5..ebf498fc1b8 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt @@ -48,7 +48,7 @@ open class VariableDeclaration : ValueDeclaration(), HasInitializer, HasType.Typ * the [ConstructExpression] is created. */ @Relationship(value = "TEMPLATE_PARAMETERS", direction = Relationship.Direction.OUTGOING) - var templateParameterEdges = astEdgesOf(label = "TEMPLATE_PARAMETERS") + var templateParameterEdges = astEdgesOf() var templateParameters by unwrapping(VariableDeclaration::templateParameterEdges) /** Determines if this is a global variable. */ @@ -69,7 +69,6 @@ open class VariableDeclaration : ValueDeclaration(), HasInitializer, HasType.Typ @Relationship("INITIALIZER") var initializerEdge = astOptionalEdgeOf( - label = "INITIALIZER", onChanged = { old, new -> val value = new?.end exchangeTypeObserver(old, new) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt index 5e7d09dce9d..5fba6f689a8 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt @@ -42,11 +42,10 @@ open class AstEdge(start: Node, end: T, override var labels: Set Node.astEdgesOf( - label: String? = null, onAdd: ((AstEdge) -> Unit)? = null, onRemove: ((AstEdge) -> Unit)? = null, ): AstEdges> { - return AstEdges(thisRef = this, label = label, onAdd = onAdd, onRemove = onRemove) + return AstEdges(thisRef = this, onAdd = onAdd, onRemove = onRemove) } /** @@ -54,12 +53,11 @@ fun Node.astEdgesOf( * container). */ fun Node.astOptionalEdgeOf( - label: String? = null, onChanged: ((old: AstEdge?, new: AstEdge?) -> Unit)? = null ): EdgeSingletonList> { return EdgeSingletonList( thisRef = this, - init = { start, end -> AstEdge(start, end, labels = setOfNotNull(label, "AST")) }, + init = { start, end -> AstEdge(start, end) }, outgoing = true, onChanged = onChanged, of = null @@ -71,12 +69,11 @@ fun Node.astOptionalEdgeOf( */ fun Node.astEdgeOf( of: NodeType, - label: String? = null, onChanged: ((old: AstEdge?, new: AstEdge?) -> Unit)? = null, ): EdgeSingletonList> { return EdgeSingletonList( thisRef = this, - init = { start, end -> AstEdge(start, end, labels = setOfNotNull(label, "AST")) }, + init = { start, end -> AstEdge(start, end) }, outgoing = true, onChanged = onChanged, of = of @@ -86,12 +83,11 @@ fun Node.astEdgeOf( /** This property edge list describes elements that are AST children of a node. */ open class AstEdges>( thisRef: Node, - label: String? = null, onAdd: ((PropertyEdgeType) -> Unit)? = null, onRemove: ((PropertyEdgeType) -> Unit)? = null, @Suppress("UNCHECKED_CAST") init: (start: Node, end: NodeType) -> PropertyEdgeType = { start, end -> - AstEdge(start, end, labels = setOfNotNull(label, "AST")) as PropertyEdgeType + AstEdge(start, end) as PropertyEdgeType }, ) : EdgeList( diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/TemplateArgument.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/TemplateArgument.kt index 46374d959c9..9e4518158aa 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/TemplateArgument.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/TemplateArgument.kt @@ -33,14 +33,12 @@ import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression class TemplateArgument( start: Node, end: NodeType, - label: String, var instantiation: TemplateInitialization? = TemplateInitialization.EXPLICIT, -) : AstEdge(start, end, setOf(label, "AST")) +) : AstEdge(start, end) /** A container for [TemplateArgument] edges. */ -class TemplateArguments(thisRef: Node, label: String) : +class TemplateArguments(thisRef: Node) : AstEdges>( thisRef, - label, - init = { start, end -> TemplateArgument(start, end, label = label) } + init = { start, end -> TemplateArgument(start, end) } ) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/AssertStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/AssertStatement.kt index 2974aa06175..bc9910b4295 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/AssertStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/AssertStatement.kt @@ -33,13 +33,11 @@ import org.neo4j.ogm.annotation.Relationship /** Represents an assert statement */ class AssertStatement : Statement() { - @Relationship(value = "CONDITION") - var conditionEdge = astOptionalEdgeOf(label = "CONDITION") + @Relationship(value = "CONDITION") var conditionEdge = astOptionalEdgeOf() /** The condition to be evaluated. */ var condition by unwrapping(AssertStatement::conditionEdge) - @Relationship(value = "MESSAGE") - var messageEdge = astOptionalEdgeOf(label = "MESSAGE") + @Relationship(value = "MESSAGE") var messageEdge = astOptionalEdgeOf() /** The *optional* message that is shown, if the assert is evaluated as true */ var message by unwrapping(AssertStatement::messageEdge) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CaseStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CaseStatement.kt index 1c65983ddb4..e0dfe6f36f9 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CaseStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CaseStatement.kt @@ -39,7 +39,7 @@ import org.neo4j.ogm.annotation.Relationship */ class CaseStatement : Statement() { @Relationship(value = "CASE_EXPRESSION") - var caseExpressionEdge = astOptionalEdgeOf(label = "CASE_EXPRESSION") + var caseExpressionEdge = astOptionalEdgeOf() /** * Primitive side effect free statement that has to match with the evaluated selector in diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CatchClause.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CatchClause.kt index ba52c16d691..7e93010f707 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CatchClause.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/CatchClause.kt @@ -37,12 +37,11 @@ import java.util.Objects import org.neo4j.ogm.annotation.Relationship class CatchClause : Statement(), BranchingNode, EOGStarterHolder { - @Relationship(value = "PARAMETER") - var parameterEdge = astOptionalEdgeOf(label = "PARAMETER") + @Relationship(value = "PARAMETER") var parameterEdge = astOptionalEdgeOf() var parameter by unwrapping(CatchClause::parameterEdge) - @Relationship(value = "BODY") var bodyEdge = astOptionalEdgeOf(label = "BODY") + @Relationship(value = "BODY") var bodyEdge = astOptionalEdgeOf() var body by unwrapping(CatchClause::bodyEdge) @DoNotPersist diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.kt index dd16600207b..31285ccc599 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.kt @@ -45,7 +45,7 @@ open class DeclarationStatement : Statement() { * it only contains a single [Declaration]. */ @Relationship(value = "DECLARATIONS", direction = Relationship.Direction.OUTGOING) - var declarationEdges = astEdgesOf(label = "DECLARATIONS") + var declarationEdges = astEdgesOf() override var declarations by unwrapping(DeclarationStatement::declarationEdges) var singleDeclaration: Declaration? diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DoStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DoStatement.kt index 24711e2e2ea..256546f5519 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DoStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/DoStatement.kt @@ -39,8 +39,7 @@ import org.neo4j.ogm.annotation.Relationship * a [Block], is executed and re-executed if the [condition] evaluates to true. */ class DoStatement : LoopStatement(), ArgumentHolder { - @Relationship("CONDITION") - var conditionEdge = astOptionalEdgeOf(label = "CONDITION") + @Relationship("CONDITION") var conditionEdge = astOptionalEdgeOf() /** * The loop condition that is evaluated after the loop statement and may trigger reevaluation. */ diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForEachStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForEachStatement.kt index 2f6b9d3f6f7..ed47af14a64 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForEachStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForEachStatement.kt @@ -45,7 +45,6 @@ class ForEachStatement : LoopStatement(), BranchingNode, StatementHolder { @Relationship("VARIABLE") var variableEdge = astOptionalEdgeOf( - label = "VARIABLE", onChanged = { _, new -> val end = new?.end if (end is Reference) { @@ -60,7 +59,7 @@ class ForEachStatement : LoopStatement(), BranchingNode, StatementHolder { */ var variable by unwrapping(ForEachStatement::variableEdge) - @Relationship("ITERABLE") var iterableEdge = astOptionalEdgeOf(label = "ITERABLE") + @Relationship("ITERABLE") var iterableEdge = astOptionalEdgeOf() /** This field contains the iteration subject of the loop. */ var iterable by unwrapping(ForEachStatement::iterableEdge) @@ -69,7 +68,7 @@ class ForEachStatement : LoopStatement(), BranchingNode, StatementHolder { override var statementEdges: AstEdges> get() { - val statements = astEdgesOf(label = "STATEMENTS") + val statements = astEdgesOf() statements += variableEdge statements += iterableEdge statements += statementEdge diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForStatement.kt index 4e59de9266a..482c7a32c76 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ForStatement.kt @@ -42,19 +42,17 @@ import org.neo4j.ogm.annotation.Relationship class ForStatement : LoopStatement(), BranchingNode { @Relationship("INITIALIZER_STATEMENT") - var initializerStatementEdge = astOptionalEdgeOf(label = "INITIALIZER_STATEMENT") + var initializerStatementEdge = astOptionalEdgeOf() var initializerStatement by unwrapping(ForStatement::initializerStatementEdge) @Relationship("CONDITION_DECLARATION") - var conditionDeclarationEdge = astOptionalEdgeOf(label = "CONDITION_DECLARATION") + var conditionDeclarationEdge = astOptionalEdgeOf() var conditionDeclaration by unwrapping(ForStatement::conditionDeclarationEdge) - @Relationship("CONDITION") - var conditionEdge = astOptionalEdgeOf(label = "CONDITION") + @Relationship("CONDITION") var conditionEdge = astOptionalEdgeOf() var condition by unwrapping(ForStatement::conditionEdge) - @Relationship("ITERATION_STATEMENT") - var iterationStatementEdge = astOptionalEdgeOf(label = "ITERATION_STATEMENT") + @Relationship("ITERATION_STATEMENT") var iterationStatementEdge = astOptionalEdgeOf() var iterationStatement by unwrapping(ForStatement::iterationStatementEdge) override val branchedBy: Node? diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/IfStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/IfStatement.kt index dfdc2f9d05e..346d78b533f 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/IfStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/IfStatement.kt @@ -40,17 +40,16 @@ import org.neo4j.ogm.annotation.Relationship /** Represents a condition control flow statement, usually indicating by `If`. */ class IfStatement : Statement(), BranchingNode, ArgumentHolder { @Relationship(value = "INITIALIZER_STATEMENT") - var initializerStatementEdge = astOptionalEdgeOf(label = "INITIALIZER_STATEMENT") + var initializerStatementEdge = astOptionalEdgeOf() /** C++ initializer statement. */ var initializerStatement by unwrapping(IfStatement::initializerStatementEdge) @Relationship(value = "CONDITION_DECLARATION") - var conditionDeclarationEdge = astOptionalEdgeOf(label = "CONDITION_DECLARATION") + var conditionDeclarationEdge = astOptionalEdgeOf() /** C++ alternative to the condition. */ var conditionDeclaration by unwrapping(IfStatement::conditionDeclarationEdge) - @Relationship(value = "CONDITION") - var conditionEdge = astOptionalEdgeOf(label = "CONDITION") + @Relationship(value = "CONDITION") var conditionEdge = astOptionalEdgeOf() /** The condition to be evaluated. */ var condition by unwrapping(IfStatement::conditionEdge) @@ -60,13 +59,11 @@ class IfStatement : Statement(), BranchingNode, ArgumentHolder { /** C++ constexpr construct. */ var isConstExpression = false - @Relationship(value = "THEN_STATEMENT") - var thenStatementEdge = astOptionalEdgeOf(label = "THEN_STATEMENT") + @Relationship(value = "THEN_STATEMENT") var thenStatementEdge = astOptionalEdgeOf() /** The statement that is executed, if the condition is evaluated as true. Usually a [Block]. */ var thenStatement by unwrapping(IfStatement::thenStatementEdge) - @Relationship(value = "ELSE_STATEMENT") - var elseStatementEdge = astOptionalEdgeOf(label = "ELSE_STATEMENT") + @Relationship(value = "ELSE_STATEMENT") var elseStatementEdge = astOptionalEdgeOf() /** * The statement that is executed, if the condition is evaluated as false. Usually a [Block]. */ diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/LabelStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/LabelStatement.kt index 0eff2e9b7a7..88eddde61cb 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/LabelStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/LabelStatement.kt @@ -40,8 +40,7 @@ import org.neo4j.ogm.annotation.Relationship * breaks (Java) or goto(C++). */ class LabelStatement : Statement(), StatementHolder { - @Relationship(value = "SUB_STATEMENT") - var subStatementEdge = astOptionalEdgeOf(label = "SUB_STATEMENT") + @Relationship(value = "SUB_STATEMENT") var subStatementEdge = astOptionalEdgeOf() /** Statement that the label is attached to. Can be a simple or compound statement. */ var subStatement by unwrapping(LabelStatement::subStatementEdge) @@ -59,7 +58,7 @@ class LabelStatement : Statement(), StatementHolder { override var statementEdges: AstEdges> get() { - var list = astEdgesOf(label = "STATEMENTS") + var list = astEdgesOf() subStatement?.let { list.resetTo(listOf(it)) } return list } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/LoopStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/LoopStatement.kt index 13b551faeb6..5db8239930a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/LoopStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/LoopStatement.kt @@ -43,7 +43,7 @@ import org.neo4j.ogm.annotation.Relationship */ abstract class LoopStatement : Statement() { - @Relationship("STATEMENT") var statementEdge = astOptionalEdgeOf(label = "STATEMENT") + @Relationship("STATEMENT") var statementEdge = astOptionalEdgeOf() /** This field contains the body of the loop, e.g. a [Block] or single [Statement]. */ var statement by unwrapping(LoopStatement::statementEdge) @@ -54,8 +54,7 @@ abstract class LoopStatement : Statement() { * `else`-Statement at loop level. E.g. in Python the [elseStatement] is executed when the loop * was not left through a break. */ - @Relationship(value = "ELSE_STATEMENT") - var elseStatementEdge = astOptionalEdgeOf(label = "ELSE_STATEMENT") + @Relationship(value = "ELSE_STATEMENT") var elseStatementEdge = astOptionalEdgeOf() var elseStatement by unwrapping(LoopStatement::elseStatementEdge) override fun toString() = diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ReturnStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ReturnStatement.kt index f5ba10c00ed..420db4def05 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ReturnStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ReturnStatement.kt @@ -35,8 +35,7 @@ import org.neo4j.ogm.annotation.Relationship /** Represents a statement that returns out of the current function. */ class ReturnStatement : Statement(), ArgumentHolder { - @Relationship(value = "RETURN_VALUES") - var returnValueEdges = astEdgesOf(label = "RETURN_VALUES") + @Relationship(value = "RETURN_VALUES") var returnValueEdges = astEdgesOf() /** The expression whose value will be returned. */ var returnValues by unwrapping(ReturnStatement::returnValueEdges) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/Statement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/Statement.kt index 129c3ddb2d8..f006d9c132e 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/Statement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/Statement.kt @@ -52,7 +52,7 @@ abstract class Statement : Node(), DeclarationHolder { * TODO: This is actually an AST node just for a subset of nodes, i.e. initializers in for-loops */ @Relationship(value = "LOCALS", direction = Relationship.Direction.OUTGOING) - var localEdges = astEdgesOf(label = "LOCALS") + var localEdges = astEdgesOf() /** Virtual property to access [localEdges] without property edges. */ var locals by unwrapping(Statement::localEdges) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SwitchStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SwitchStatement.kt index 3277d36cb19..9d955a50575 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SwitchStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SwitchStatement.kt @@ -40,23 +40,21 @@ import org.neo4j.ogm.annotation.Relationship * handled properly. */ class SwitchStatement : Statement(), BranchingNode { - @Relationship(value = "SELECTOR") - var selectorEdge = astOptionalEdgeOf(label = "SELECTOR") + @Relationship(value = "SELECTOR") var selectorEdge = astOptionalEdgeOf() /** Selector that determines the case/default statement of the subsequent execution */ var selector by unwrapping(SwitchStatement::selectorEdge) @Relationship(value = "INITIALIZER_STATEMENT") - var initializerStatementEdge = astOptionalEdgeOf(label = "INITIALIZER_STATEMENT") + var initializerStatementEdge = astOptionalEdgeOf() /** C++ can have an initializer statement in a switch */ var initializerStatement by unwrapping(SwitchStatement::initializerStatementEdge) @Relationship(value = "SELECTOR_DECLARATION") - var selectorDeclarationEdge = astOptionalEdgeOf(label = "SELECTOR_DECLARATION") + var selectorDeclarationEdge = astOptionalEdgeOf() /** C++ allows to use a declaration instead of an expression as selector */ var selectorDeclaration by unwrapping(SwitchStatement::selectorDeclarationEdge) - @Relationship(value = "STATEMENT") - var statementEdge = astOptionalEdgeOf(label = "STATEMENT") + @Relationship(value = "STATEMENT") var statementEdge = astOptionalEdgeOf() /** * The compound statement that contains break/default statements with regular statements on the * same hierarchy diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SynchronizedStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SynchronizedStatement.kt index 0608e511552..ac955b24ce6 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SynchronizedStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/SynchronizedStatement.kt @@ -33,11 +33,10 @@ import java.util.Objects import org.neo4j.ogm.annotation.Relationship class SynchronizedStatement : Statement() { - @Relationship(value = "EXPRESSION") - var expressionEdge = astOptionalEdgeOf(label = "EXPRESSION") + @Relationship(value = "EXPRESSION") var expressionEdge = astOptionalEdgeOf() var expression by unwrapping(SynchronizedStatement::expressionEdge) - @Relationship(value = "BLOCK") var blockEdge = astOptionalEdgeOf(label = "BLOCK") + @Relationship(value = "BLOCK") var blockEdge = astOptionalEdgeOf() var block by unwrapping(SynchronizedStatement::blockEdge) override fun equals(other: Any?): Boolean { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ThrowExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ThrowExpression.kt index 59d245f79b3..0f8b86cb32a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ThrowExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/ThrowExpression.kt @@ -37,8 +37,7 @@ import org.neo4j.ogm.annotation.Relationship class ThrowExpression : Expression(), ArgumentHolder { /** The exception object to be raised. */ - @Relationship(value = "EXCEPTION") - var exceptionEdge = astOptionalEdgeOf(label = "EXCEPTION") + @Relationship(value = "EXCEPTION") var exceptionEdge = astOptionalEdgeOf() var exception by unwrapping(ThrowExpression::exceptionEdge) /** @@ -46,7 +45,7 @@ class ThrowExpression : Expression(), ArgumentHolder { * was raised while handling another exception. */ @Relationship(value = "PARENT_EXCEPTION") - var parentExceptionEdge = astOptionalEdgeOf(label = "PARENT_EXCEPTION") + var parentExceptionEdge = astOptionalEdgeOf() var parentException by unwrapping(ThrowExpression::parentExceptionEdge) override fun addArgument(expression: Expression) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/TryStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/TryStatement.kt index cd96ca44d31..1f887fa2075 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/TryStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/TryStatement.kt @@ -42,15 +42,14 @@ class TryStatement : Statement() { * enter the [tryBlock]. */ @Relationship(value = "RESOURCES", direction = Relationship.Direction.OUTGOING) - var resourceEdges = astEdgesOf(label = "RESOURCES") + var resourceEdges = astEdgesOf() var resources by unwrapping(TryStatement::resourceEdges) /** * This represents a block whose statements can throw exceptions which are handled by the * [catchClauses]. */ - @Relationship(value = "TRY_BLOCK") - var tryBlockEdge = astOptionalEdgeOf(label = "TRY_BLOCK") + @Relationship(value = "TRY_BLOCK") var tryBlockEdge = astOptionalEdgeOf() var tryBlock by unwrapping(TryStatement::tryBlockEdge) /** @@ -58,8 +57,7 @@ class TryStatement : Statement() { * exceptions. Note that any exception thrown in this block is no longer caught by the * [catchClauses]. */ - @Relationship(value = "ELSE_BLOCK") - var elseBlockEdge = astOptionalEdgeOf(label = "ELSE_BLOCK") + @Relationship(value = "ELSE_BLOCK") var elseBlockEdge = astOptionalEdgeOf() var elseBlock by unwrapping(TryStatement::elseBlockEdge) /** @@ -67,8 +65,7 @@ class TryStatement : Statement() { * or one of the [catchClauses]. Note that any exception thrown in this block is no longer * caught by the [catchClauses]. */ - @Relationship(value = "FINALLY_BLOCK") - var finallyBlockEdge = astOptionalEdgeOf(label = "FINALLY_BLOCK") + @Relationship(value = "FINALLY_BLOCK") var finallyBlockEdge = astOptionalEdgeOf() var finallyBlock by unwrapping(TryStatement::finallyBlockEdge) /** @@ -77,7 +74,7 @@ class TryStatement : Statement() { * exists. */ @Relationship(value = "CATCH_CLAUSES", direction = Relationship.Direction.OUTGOING) - var catchClauseEdges = astEdgesOf(label = "CATCH_CLAUSES") + var catchClauseEdges = astEdgesOf() var catchClauses by unwrapping(TryStatement::catchClauseEdges) override fun equals(other: Any?): Boolean { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/WhileStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/WhileStatement.kt index ffb9d38cc6e..5c3bfb69b41 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/WhileStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/WhileStatement.kt @@ -42,12 +42,11 @@ import org.neo4j.ogm.annotation.Relationship */ class WhileStatement : LoopStatement(), BranchingNode, ArgumentHolder { @Relationship(value = "CONDITION_DECLARATION") - var conditionDeclarationEdge = astOptionalEdgeOf(label = "CONDITION_DECLARATION") + var conditionDeclarationEdge = astOptionalEdgeOf() /** C++ allows defining a declaration instead of a pure logical expression as condition */ var conditionDeclaration by unwrapping(WhileStatement::conditionDeclarationEdge) - @Relationship(value = "CONDITION") - var conditionEdge = astOptionalEdgeOf(label = "CONDITION") + @Relationship(value = "CONDITION") var conditionEdge = astOptionalEdgeOf() /** The condition that decides if the block is executed. */ var condition by unwrapping(WhileStatement::conditionEdge) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt index c1f335c565b..89ae115c2ae 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt @@ -63,7 +63,6 @@ class AssignExpression : @Relationship("LHS") var lhsEdges = astEdgesOf( - label = "LHS", onAdd = { var end = it.end var base = (end as? MemberExpression)?.base as? MemberExpression @@ -84,11 +83,9 @@ class AssignExpression : @Relationship("RHS") /** The expressions on the right-hand side. */ var rhsEdges = - astEdgesOf( - label = "RHS", - onAdd = { it.end.registerTypeObserver(this) }, - onRemove = { it.end.unregisterTypeObserver(this) } - ) + astEdgesOf(onAdd = { it.end.registerTypeObserver(this) }) { + it.end.unregisterTypeObserver(this) + } var rhs by unwrapping(AssignExpression::rhsEdges) /** @@ -126,8 +123,7 @@ class AssignExpression : return operatorCode in (language?.simpleAssignmentOperators ?: setOf()) } - @Relationship("DECLARATIONS") - var declarationEdges = astEdgesOf(label = "DECLARATIONS") + @Relationship("DECLARATIONS") var declarationEdges = astEdgesOf() /** * Some languages, such as Go explicitly allow the definition / declaration of variables in the * assignment (known as a "short assignment"). Some languages, such as Python even implicitly diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/BinaryOperator.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/BinaryOperator.kt index 63e1ccec0d4..06cce9f4221 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/BinaryOperator.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/BinaryOperator.kt @@ -50,7 +50,6 @@ open class BinaryOperator : var lhsEdge = astEdgeOf( of = ProblemExpression("could not parse lhs"), - label = "LHS", onChanged = ::exchangeTypeObserver ) var lhs by unwrapping(BinaryOperator::lhsEdge) @@ -60,7 +59,6 @@ open class BinaryOperator : var rhsEdge = astEdgeOf( of = ProblemExpression("could not parse rhs"), - label = "RHS", onChanged = ::exchangeTypeObserver ) var rhs by unwrapping(BinaryOperator::rhsEdge) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Block.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Block.kt index 16ad6ad0f33..0677c750870 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Block.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Block.kt @@ -42,7 +42,7 @@ import org.neo4j.ogm.annotation.Relationship open class Block : Expression(), StatementHolder { /** The list of statements. */ @Relationship(value = "STATEMENTS", direction = Relationship.Direction.OUTGOING) - override var statementEdges = astEdgesOf(label = "STATEMENTS") + override var statementEdges = astEdgesOf() override var statements by unwrapping(Block::statementEdges) /** * This variable helps to differentiate between static and non-static initializer blocks. Static diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt index e33affe188e..221d6864323 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt @@ -65,7 +65,7 @@ open class CallExpression : /** The list of arguments of this call expression, backed by a list of [Edge] objects. */ @Relationship(value = "ARGUMENTS", direction = Relationship.Direction.OUTGOING) - var argumentEdges = astEdgesOf(label = "ARGUMENTS") + var argumentEdges = astEdgesOf() /** * The list of arguments as a simple list. This is a delegated property delegated to @@ -85,8 +85,7 @@ open class CallExpression : * is intentionally left empty. It is not filled by the [SymbolResolver]. */ @Relationship(value = "CALLEE", direction = Relationship.Direction.OUTGOING) - private var calleeEdge = - astEdgeOf(ProblemExpression("could not parse callee"), label = "CALLEE") + private var calleeEdge = astEdgeOf(ProblemExpression("could not parse callee")) var callee by unwrapping(CallExpression::calleeEdge) @@ -186,7 +185,7 @@ open class CallExpression : ) { if (templateParam is Expression || templateParam is Type) { if (templateArgumentEdges == null) { - templateArgumentEdges = TemplateArguments(this, label = "TEMPLATE_ARGUMENTS") + templateArgumentEdges = TemplateArguments(this) } templateArgumentEdges?.add(templateParam) { instantiation = templateInitialization } @@ -199,7 +198,7 @@ open class CallExpression : orderedInitializationSignature: List ) { if (templateArgumentEdges == null) { - templateArgumentEdges = TemplateArguments(this, label = "TEMPLATE_ARGUMENTS") + templateArgumentEdges = TemplateArguments(this) } for (edge in templateArgumentEdges ?: listOf()) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CastExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CastExpression.kt index 19f9f54eceb..541cf38a255 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CastExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CastExpression.kt @@ -48,7 +48,6 @@ class CastExpression : Expression(), ArgumentHolder, HasType.TypeObserver { var expressionEdge = astEdgeOf( of = ProblemExpression("could not parse inner expression"), - label = "EXPRESSION", onChanged = ::exchangeTypeObserver ) var expression by unwrapping(CastExpression::expressionEdge) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CollectionComprehension.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CollectionComprehension.kt index d75a716c2f1..2e99cb4deae 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CollectionComprehension.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CollectionComprehension.kt @@ -46,8 +46,7 @@ import org.neo4j.ogm.annotation.Relationship class CollectionComprehension : Expression(), ArgumentHolder { @Relationship("COMPREHENSION_EXPRESSIONS") - var comprehensionExpressionEdges = - astEdgesOf(label = "COMPREHENSION_EXPRESSIONS") + var comprehensionExpressionEdges = astEdgesOf() /** * This field contains one or multiple [ComprehensionExpression]s. * @@ -62,7 +61,6 @@ class CollectionComprehension : Expression(), ArgumentHolder { var statementEdge = astEdgeOf( ProblemExpression("No statement provided but is required in ${this::class}"), - label = "STATEMENT", ) /** * This field contains the statement which is applied to each element of the input for which the diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ComprehensionExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ComprehensionExpression.kt index 4b8f87158d2..aeb24c0bf20 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ComprehensionExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ComprehensionExpression.kt @@ -41,7 +41,6 @@ class ComprehensionExpression : Expression(), ArgumentHolder { var variableEdge = astEdgeOf( of = ProblemExpression("Missing variableEdge in ${this::class}"), - label = "VARIABLE", onChanged = { _, new -> val end = new?.end if (end is Reference) { @@ -58,15 +57,12 @@ class ComprehensionExpression : Expression(), ArgumentHolder { @Relationship("ITERABLE") var iterableEdge = - astEdgeOf( - ProblemExpression("Missing iterable in ${this::class}"), - label = "ITERABLE" - ) + astEdgeOf(ProblemExpression("Missing iterable in ${this::class}")) /** This field contains the iteration subject of the loop. */ var iterable by unwrapping(ComprehensionExpression::iterableEdge) - @Relationship("PREDICATE") var predicateEdge = astOptionalEdgeOf(label = "PREDICATE") + @Relationship("PREDICATE") var predicateEdge = astOptionalEdgeOf() /** * This field contains the predicate which has to hold to evaluate `statement(variable)` and diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConditionalExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConditionalExpression.kt index bc8b0a85e8d..977c85d8ed3 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConditionalExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConditionalExpression.kt @@ -43,16 +43,12 @@ import org.neo4j.ogm.annotation.Relationship class ConditionalExpression : Expression(), ArgumentHolder, BranchingNode, HasType.TypeObserver { @Relationship("CONDITION") var conditionEdge = - astEdgeOf( - ProblemExpression("could not parse condition expression"), - label = "CONDITION" - ) + astEdgeOf(ProblemExpression("could not parse condition expression")) var condition by unwrapping(ConditionalExpression::conditionEdge) @Relationship("THEN_EXPRESSION") var thenExpressionEdge = astOptionalEdgeOf( - label = "THEN_EXPRESSION", onChanged = { old, new -> old?.end?.unregisterTypeObserver(this) new?.end?.registerTypeObserver(this) @@ -63,7 +59,6 @@ class ConditionalExpression : Expression(), ArgumentHolder, BranchingNode, HasTy @Relationship("ELSE_EXPRESSION") var elseExpressionEdge = astOptionalEdgeOf( - label = "ELSE_EXPRESSION", onChanged = { old, new -> old?.end?.unregisterTypeObserver(this) new?.end?.registerTypeObserver(this) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConstructExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConstructExpression.kt index c9f3d3535aa..c6c81d51ae8 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConstructExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConstructExpression.kt @@ -65,8 +65,7 @@ class ConstructExpression : CallExpression() { } } - @Relationship("ANONYMOUS_CLASS") - var anonymousClassEdge = astOptionalEdgeOf(label = "ANONYMOUS_CLASS") + @Relationship("ANONYMOUS_CLASS") var anonymousClassEdge = astOptionalEdgeOf() var anonymousClass by unwrapping(ConstructExpression::anonymousClassEdge) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/DeleteExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/DeleteExpression.kt index 2e969393950..dfa39b08f2f 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/DeleteExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/DeleteExpression.kt @@ -31,7 +31,7 @@ import java.util.Objects import org.neo4j.ogm.annotation.Relationship class DeleteExpression : Expression() { - @Relationship("OPERANDS") var operandEdges = astEdgesOf(label = "OPERANDS") + @Relationship("OPERANDS") var operandEdges = astEdgesOf() var operands by unwrapping(DeleteExpression::operandEdges) override fun equals(other: Any?): Boolean { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ExpressionList.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ExpressionList.kt index be7e87711d5..f349cc7a332 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ExpressionList.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ExpressionList.kt @@ -34,7 +34,7 @@ import org.neo4j.ogm.annotation.Relationship class ExpressionList : Expression() { @Relationship(value = "SUBEXPR", direction = Relationship.Direction.OUTGOING) - var expressionEdges = astEdgesOf(label = "SUBEXPR") + var expressionEdges = astEdgesOf() var expressions by unwrapping(ExpressionList::expressionEdges) override fun equals(other: Any?): Boolean { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt index bf04043718a..2f70d1026f1 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt @@ -49,9 +49,9 @@ class InitializerListExpression : Expression(), ArgumentHolder, HasType.TypeObse var initializerEdges = astEdgesOf( onAdd = { it.end.registerTypeObserver(this) }, - onRemove = { it.end.unregisterTypeObserver(this) }, - label = "INITIALIZERS", - ) + ) { + it.end.unregisterTypeObserver(this) + } /** Virtual property to access [initializerEdges] without property edges. */ var initializers by unwrapping(InitializerListExpression::initializerEdges) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/KeyValueExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/KeyValueExpression.kt index 04325a7d815..d6b2eaff816 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/KeyValueExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/KeyValueExpression.kt @@ -40,16 +40,14 @@ import org.neo4j.ogm.annotation.Relationship */ class KeyValueExpression : Expression(), ArgumentHolder { - @Relationship("KEY") - var keyEdge = astEdgeOf(ProblemExpression("missing key"), label = "KEY") + @Relationship("KEY") var keyEdge = astEdgeOf(ProblemExpression("missing key")) /** * The key of this pair. It is usually a literal, but some languages even allow references to * variables as a key. */ var key by unwrapping(KeyValueExpression::keyEdge) - @Relationship("VALUE") - var valueEdge = astEdgeOf(ProblemExpression("missing value"), label = "VALUE") + @Relationship("VALUE") var valueEdge = astEdgeOf(ProblemExpression("missing value")) /** The value of this pair. It can be any expression */ var value by unwrapping(KeyValueExpression::valueEdge) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/LambdaExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/LambdaExpression.kt index 5827f3ebe1b..2b8823be979 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/LambdaExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/LambdaExpression.kt @@ -51,11 +51,7 @@ class LambdaExpression : Expression(), HasType.TypeObserver { var areVariablesMutable: Boolean = true @Relationship("FUNCTION") - var functionEdge = - astOptionalEdgeOf( - label = "FUNCTION", - onChanged = ::exchangeTypeObserver - ) + var functionEdge = astOptionalEdgeOf(onChanged = ::exchangeTypeObserver) var function by unwrapping(LambdaExpression::functionEdge) override fun typeChanged(newType: Type, src: HasType) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.kt index bee94e37e58..21c0e2cb85c 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.kt @@ -48,7 +48,6 @@ class MemberExpression : Reference(), HasOverloadedOperation, ArgumentHolder, Ha var baseEdge = astEdgeOf( ProblemExpression("could not parse base expression"), - label = "BASE", onChanged = { old, new -> exchangeTypeObserver(old, new) updateName() diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/NewArrayExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/NewArrayExpression.kt index 4f4ad5d0fd7..d0bfa683f22 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/NewArrayExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/NewArrayExpression.kt @@ -39,8 +39,7 @@ import org.neo4j.ogm.annotation.Relationship */ // TODO Merge and/or refactor with new Expression class NewArrayExpression : Expression() { - @Relationship("INITIALIZER") - var initializerEdge = astOptionalEdgeOf(label = "INITIALIZER") + @Relationship("INITIALIZER") var initializerEdge = astOptionalEdgeOf() /** * The initializer of the expression, if present. Many languages, such as Java, either specify @@ -54,7 +53,7 @@ class NewArrayExpression : Expression() { * dimensions. In the graph, this will NOT be done. */ @Relationship(value = "DIMENSIONS", direction = Relationship.Direction.OUTGOING) - var dimensionEdges = astEdgesOf(label = "DIMENSIONS") + var dimensionEdges = astEdgesOf() /** Virtual property to access [dimensionEdges] without property edges. */ var dimensions by unwrapping(NewArrayExpression::dimensionEdges) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/NewExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/NewExpression.kt index 9132992c405..807dbcdf296 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/NewExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/NewExpression.kt @@ -36,8 +36,7 @@ import org.neo4j.ogm.annotation.Relationship /** Represents the creation of a new object through the `new` keyword. */ class NewExpression : Expression(), HasInitializer { - @Relationship("INITIALIZER") - var initializerEdge = astOptionalEdgeOf(label = "INITIALIZER") + @Relationship("INITIALIZER") var initializerEdge = astOptionalEdgeOf() /** The initializer expression. */ override var initializer by unwrapping(NewExpression::initializerEdge) @@ -47,7 +46,7 @@ class NewExpression : Expression(), HasInitializer { * ConstructExpression is created */ @Relationship(value = "TEMPLATE_PARAMETERS", direction = Relationship.Direction.OUTGOING) - var templateParameterEdges = astEdgesOf(label = "TEMPLATE_PARAMETERS") + var templateParameterEdges = astEdgesOf() var templateParameters by unwrapping(NewExpression::templateParameterEdges) override fun toString(): String { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/RangeExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/RangeExpression.kt index 180bdc84801..0140852024c 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/RangeExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/RangeExpression.kt @@ -55,15 +55,15 @@ import org.neo4j.ogm.annotation.Relationship * Individual meaning of the range indices might differ per language. */ class RangeExpression : Expression() { - @Relationship("FLOOR") var floorEdge = astOptionalEdgeOf(label = "FLOOR") + @Relationship("FLOOR") var floorEdge = astOptionalEdgeOf() /** The lower bound ("floor") of the range. This index is usually *inclusive*. */ var floor by unwrapping(RangeExpression::floorEdge) - @Relationship("CEILING") var ceilingEdge = astOptionalEdgeOf(label = "CEILING") + @Relationship("CEILING") var ceilingEdge = astOptionalEdgeOf() /** The upper bound ("ceiling") of the range. This index is usually *exclusive*. */ var ceiling by unwrapping(RangeExpression::ceilingEdge) - @Relationship("THIRD") var thirdEdge = astOptionalEdgeOf(label = "THIRD") + @Relationship("THIRD") var thirdEdge = astOptionalEdgeOf() /** * Some languages offer a third value. The meaning depends completely on the language. For * example, Python allows specifying a step, while Go allows to control the underlying array's diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/SubscriptExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/SubscriptExpression.kt index b4b3a70d552..8254241a749 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/SubscriptExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/SubscriptExpression.kt @@ -43,7 +43,6 @@ class SubscriptExpression : Expression(), HasBase, HasType.TypeObserver, Argumen var arrayExpressionEdge = astEdgeOf( of = ProblemExpression("could not parse array expression"), - label = "ARRAY_EXPRESSION", onChanged = ::exchangeTypeObserver ) /** The array on which the access is happening. This is most likely a [Reference]. */ @@ -51,10 +50,7 @@ class SubscriptExpression : Expression(), HasBase, HasType.TypeObserver, Argumen @Relationship("SUBSCRIPT_EXPRESSION") var subscriptExpressionEdge = - astEdgeOf( - ProblemExpression("could not parse index expression"), - label = "SUBSCRIPT_EXPRESSION" - ) + astEdgeOf(ProblemExpression("could not parse index expression")) /** * The expression which represents the "subscription" or index on which the array is accessed. * This can for example be a reference to another variable ([Reference]), a [Literal] or a diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/UnaryOperator.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/UnaryOperator.kt index cfab59fc844..3bb1931a570 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/UnaryOperator.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/UnaryOperator.kt @@ -42,7 +42,6 @@ class UnaryOperator : Expression(), HasOverloadedOperation, ArgumentHolder, HasT var inputEdge = astEdgeOf( of = ProblemExpression("could not parse input"), - label = "INPUT", onChanged = { old, new -> exchangeTypeObserver(old, new) changeExpressionAccess() diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Neo4J.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Neo4J.kt index 09f6c2fb674..2e25a93e9f4 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Neo4J.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Neo4J.kt @@ -28,6 +28,7 @@ package de.fraunhofer.aisec.cpg.persistence import de.fraunhofer.aisec.cpg.TranslationResult import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.Persistable +import de.fraunhofer.aisec.cpg.graph.edges.Edge import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeCollection import de.fraunhofer.aisec.cpg.graph.nodes import de.fraunhofer.aisec.cpg.helpers.Benchmark @@ -84,8 +85,8 @@ fun TranslationResult.persist() { val b = Benchmark(Persistable::class.java, "Persisting translation result") val astNodes = this@persist.nodes - val connected = astNodes.flatMap { it.connectedNodes } - val nodes = astNodes + connected + val connected = astNodes.flatMap { it.connectedNodes }.toSet() + val nodes = (astNodes + connected).distinct() log.info( "Persisting {} nodes: AST nodes ({}), other nodes ({})", @@ -168,6 +169,31 @@ private fun Collection.persist() { this.chunked(edgeChunkSize).map { chunk -> createRelationships(chunk) } } +context(Session) +private fun Collection>.persistEdgesOld() { + // Create an index for the "id" field of node, because we are "MATCH"ing on it in the edge + // creation. We need to wait for this to be finished + this@Session.executeWrite { tx -> + tx.run("CREATE INDEX IF NOT EXISTS FOR (n:Node) ON (n.id)").consume() + } + + this.chunked(edgeChunkSize).map { chunk -> + createRelationships( + chunk.flatMap { edge -> + // Since Neo4J does not support multiple labels on edges, but we do internally, we + // duplicate the edge for each label + edge.labels.map { label -> + mapOf( + "startId" to edge.start.id.toString(), + "endId" to edge.end.id.toString(), + "type" to label + ) + edge.properties() + } + } + ) + } +} + /** * Creates relationships in a graph database based on provided properties. * From 866d2b3378891a89f2ca92ac041c4b2cc3ecd128 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Sat, 14 Dec 2024 00:12:59 +0100 Subject: [PATCH 20/25] Using Neo4J converter annotation for now --- .../aisec/cpg/TranslationConfiguration.kt | 2 + .../aisec/cpg/TranslationContext.kt | 2 + .../fraunhofer/aisec/cpg/TranslationResult.kt | 2 + .../aisec/cpg/helpers/MeasurementHolder.kt | 3 + .../aisec/cpg/persistence/Common.kt | 129 +++++++++++------- .../aisec/cpg/persistence/TestCommon.kt | 20 ++- 6 files changed, 104 insertions(+), 54 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt index 3c4b1906f37..bb13e63375e 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt @@ -36,6 +36,7 @@ import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.passes.* import de.fraunhofer.aisec.cpg.passes.configuration.* import de.fraunhofer.aisec.cpg.passes.inference.DFGFunctionSummaries +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import java.io.File import java.nio.file.Path import java.util.* @@ -51,6 +52,7 @@ import org.slf4j.LoggerFactory * The configuration for the [TranslationManager] holds all information that is used during the * translation. */ +@DoNotPersist class TranslationConfiguration private constructor( /** Definition of additional symbols, mostly useful for C++. */ diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationContext.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationContext.kt index 3d7636ee127..1524fe7050e 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationContext.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationContext.kt @@ -26,11 +26,13 @@ package de.fraunhofer.aisec.cpg import de.fraunhofer.aisec.cpg.graph.Component +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist /** * The translation context holds all necessary managers and configurations needed during the * translation process. */ +@DoNotPersist class TranslationContext( /** The configuration for this translation. */ val config: TranslationConfiguration, diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt index 05ace1a4dc8..7b4b3ddfb6a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt @@ -35,6 +35,7 @@ import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.helpers.MeasurementHolder import de.fraunhofer.aisec.cpg.helpers.StatisticsHolder import de.fraunhofer.aisec.cpg.passes.Pass +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import java.util.* import java.util.concurrent.ConcurrentHashMap import org.neo4j.ogm.annotation.Relationship @@ -92,6 +93,7 @@ class TranslationResult( * @return the list of all translation units. */ @Deprecated(message = "translation units of individual components should be accessed instead") + @DoNotPersist val translationUnits: List get() { if (components.size == 1) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/MeasurementHolder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/MeasurementHolder.kt index f16b284cdc8..2072ccae8f9 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/MeasurementHolder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/MeasurementHolder.kt @@ -28,6 +28,7 @@ package de.fraunhofer.aisec.cpg.helpers import com.fasterxml.jackson.databind.SerializationFeature import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import de.fraunhofer.aisec.cpg.TranslationConfiguration +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import java.io.File import java.nio.file.Path import java.time.Duration @@ -36,6 +37,7 @@ import java.util.* import org.slf4j.Logger import org.slf4j.LoggerFactory +@DoNotPersist class BenchmarkResults(val entries: List>) { val json: String @@ -175,6 +177,7 @@ constructor( } } +@DoNotPersist /** Represents some kind of measurements, e.g., on the performance or problems. */ open class MeasurementHolder @JvmOverloads diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt index d2d9a64e2ae..27a25541ae6 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt @@ -25,27 +25,19 @@ */ package de.fraunhofer.aisec.cpg.persistence -import de.fraunhofer.aisec.cpg.TranslationConfiguration -import de.fraunhofer.aisec.cpg.TranslationContext import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.Persistable import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeCollection import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeList -import de.fraunhofer.aisec.cpg.graph.edges.flows.DependenceType -import de.fraunhofer.aisec.cpg.graph.edges.flows.Granularity -import de.fraunhofer.aisec.cpg.helpers.BenchmarkResults -import de.fraunhofer.aisec.cpg.helpers.neo4j.DataflowGranularityConverter -import de.fraunhofer.aisec.cpg.helpers.neo4j.LocationConverter import de.fraunhofer.aisec.cpg.helpers.neo4j.NameConverter -import de.fraunhofer.aisec.cpg.helpers.neo4j.SimpleNameConverter -import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation import java.math.BigInteger import kotlin.reflect.KClass import kotlin.reflect.KProperty1 import kotlin.reflect.KTypeProjection import kotlin.reflect.KVariance import kotlin.reflect.KVisibility +import kotlin.reflect.full.createInstance import kotlin.reflect.full.createType import kotlin.reflect.full.hasAnnotation import kotlin.reflect.full.isSubtypeOf @@ -57,6 +49,8 @@ import kotlin.reflect.jvm.javaType import kotlin.uuid.Uuid import org.neo4j.ogm.annotation.Relationship import org.neo4j.ogm.annotation.Relationship.Direction.INCOMING +import org.neo4j.ogm.annotation.typeconversion.Convert +import org.neo4j.ogm.typeconversion.AttributeConverter import org.neo4j.ogm.typeconversion.CompositeAttributeConverter /** @@ -90,22 +84,6 @@ val schemaRelationshipCache: MutableMap, Map>> = mutableMapOf() -/** - * A set containing predefined property types represented as Kotlin type objects that can be used as - * properties in [schemaProperties]. - */ -val propertyTypes = - setOf( - String::class.createType(), - Int::class.createType(), - Long::class.createType(), - Boolean::class.createType(), - Name::class.createType(), - Uuid::class.createType(), - Granularity::class.createType(), - DependenceType::class.createType(), - ) - /** * Returns the [Persistable]'s properties. This DOES NOT include relationships, but only properties * directly attached to the node/edge. @@ -119,7 +97,7 @@ fun Persistable.properties(): Map { continue } - value.convert(entry.key, properties) + value.convert(entry, properties) } return properties @@ -127,19 +105,31 @@ fun Persistable.properties(): Map { /** * Runs any conversions that are necessary by [CompositeAttributeConverter] and - * [org.neo4j.ogm.typeconversion.AttributeConverter]. Since both of these classes are Neo4J OGM - * classes, we need to find new base types at some point. + * [AttributeConverter]. Since both of these classes are Neo4J OGM classes, we need to find new base + * types at some point. */ -fun Any.convert(originalKey: String, properties: MutableMap) { - // TODO: generalize conversions - if (this is Name && originalKey == "name") { +fun Any.convert( + entry: Map.Entry>, + properties: MutableMap +) { + val originalKey = entry.key + + val annotation = entry.value.javaField?.getAnnotation(Convert::class.java) + @Suppress("UNCHECKED_CAST") + if (annotation != null) { + val converter = annotation.value.createInstance() + if (converter is CompositeAttributeConverter<*>) { + properties += (converter as CompositeAttributeConverter).toGraphProperties(this) + } else if (converter is AttributeConverter<*, *>) { + properties.put( + originalKey, + (converter as AttributeConverter).toGraphProperty(this) + ) + } + } else if (this is Name && originalKey == "name") { + // needs to be extra because of the way annotations work, this will be re-designed once OGM + // is completely gone properties += NameConverter().toGraphProperties(this) - } else if (this is Name) { - properties.put(originalKey, SimpleNameConverter().toGraphProperty(this)) - } else if (this is PhysicalLocation) { - properties += LocationConverter().toGraphProperties(this) - } else if (this is Granularity) { - properties += DataflowGranularityConverter().toGraphProperties(this) } else if (this is Enum<*>) { properties.put(originalKey, this.name) } else if (this is Uuid) { @@ -184,15 +174,7 @@ val KClass<*>.labels: Set return labels } -/** A list of specific types that are intended to be ignored for persistence. */ -internal val ignoredTypes = - listOf( - TranslationContext::class.createType(), - TranslationConfiguration::class.createType(), - BenchmarkResults::class.createType(), - KClass::class.createType(listOf(KTypeProjection.STAR)), - ) - +internal val kClassType = KClass::class.createType(listOf(KTypeProjection.STAR)) internal val nodeType = Node::class.createType() internal val collectionType = Collection::class.createType(listOf(KTypeProjection.STAR)) internal val collectionOfNodeType = @@ -206,9 +188,8 @@ internal val mapType = Map::class.createType(listOf(KTypeProjection.STAR, KTypeP * [Persistable]. * * This property computes a map that associates property names (as strings) to their corresponding - * [KProperty1] objects, which represent the properties defined in the class. Only properties whose - * return types are included in a predefined set of supported property types ([propertyTypes]) are - * included in the map. + * [KProperty1] objects, which represent the properties defined in the class. Only properties for + * which [isSimpleProperty] returns true, are included. * * The computed map is cached to optimize subsequent lookups for properties of the same class. */ @@ -227,6 +208,25 @@ val KClass.schemaProperties: Map.schemaRelationships: Map> get() { // Check, if we already computed the relationship for this node's class @@ -249,10 +249,11 @@ val KClass.schemaRelationships: Map): Boolean val returnType = property.returnType.withNullability(false) return when { - property.hasAnnotation() -> false property.visibility == KVisibility.PRIVATE -> false - ignoredTypes.any { returnType.isSubtypeOf(it) } -> false + property.hasAnnotation() -> false + returnType.hasAnnotation() -> false + (returnType.javaType as? Class<*>)?.getAnnotation(DoNotPersist::class.java) != null -> false + returnType.isSubtypeOf(kClassType) -> false returnType.isSubtypeOf(collectionType) -> false returnType.isSubtypeOf(mapType) -> false returnType.isSubtypeOf(nodeType) -> false @@ -306,6 +309,17 @@ private fun isRelationship(property: KProperty1): Boolean { } } +/** + * Retrieves the relational name associated with a property in the context of the raph schema. + * + * The `relationshipName` is determined based on the following rules: + * - If the property is annotated with the `@Relationship` annotation, the value of the annotation + * is used as the relationship name, provided it is non-null and not an empty string. + * - If the property name ends with "Edge", this suffix is removed. This adjustment is made to + * account for cases where two variables represent an edge, one named with "Edge" and another as + * the delegate without the suffix. The desired name is the one without "Edge". + * - The resulting name is converted to UPPER_SNAKE_CASE for standardization. + */ val KProperty1.relationshipName: String get() { // If we have a (legacy) Neo4J annotation for our relationship, we take this one @@ -323,6 +337,15 @@ val KProperty1.relationshipName: String return this.name.substringBeforeLast("Edge").toUpperSnakeCase() } +/** + * Converts the current string to UPPER_SNAKE_CASE format. + * + * Each word boundary in camelCase or PascalCase naming convention is replaced with an underscore, + * and all characters are converted to uppercase. This is commonly used for representing constants + * or environment-style variable names. + * + * @return A string converted to UPPER_SNAKE_CASE. + */ fun String.toUpperSnakeCase(): String { val pattern = "(?<=.)[A-Z]".toRegex() return this.replace(pattern, "_$0").uppercase() diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/persistence/TestCommon.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/persistence/TestCommon.kt index c53fc609a10..8d68203300b 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/persistence/TestCommon.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/persistence/TestCommon.kt @@ -25,6 +25,7 @@ */ package de.fraunhofer.aisec.cpg.persistence +import de.fraunhofer.aisec.cpg.TranslationResult import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import kotlin.test.Test import kotlin.test.assertEquals @@ -54,7 +55,7 @@ class TestCommon { @Test fun testSchemaRelationships() { - val relationships = FunctionDeclaration::class.schemaRelationships + var relationships = FunctionDeclaration::class.schemaRelationships assertEquals( listOf( "ANNOTATIONS", @@ -78,5 +79,22 @@ class TestCommon { ), relationships.keys.sorted() ) + + relationships = TranslationResult::class.schemaRelationships + assertEquals( + listOf( + "ADDITIONAL_NODES", + "ANNOTATIONS", + "AST", + "CDG", + "COMPONENTS", + "DFG", + "EOG", + "LANGUAGE", + "PDG", + "SCOPE", + ), + relationships.keys.sorted() + ) } } From e05e23ae8a0571cec855d06668f648aed60cd94c Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Sat, 14 Dec 2024 00:17:37 +0100 Subject: [PATCH 21/25] Remaining code review issues --- README.md | 9 ++++++++- .../de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt | 4 +--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3a9ef5911af..98c262f3b9b 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,14 @@ Instead of manually generating or editing the `gradle.properties` file, you can ### For Visualization Purposes -In order to get familiar with the graph itself, you can use the subproject [cpg-neo4j](https://github.com/Fraunhofer-AISEC/cpg/tree/master/cpg-neo4j). It uses this library to generate the CPG for a set of user-provided code files. The graph is then persisted to a [Neo4j](https://neo4j.com/) graph database. The advantage this has for the user, is that Neo4j's visualization software [Neo4j Browser](https://neo4j.com/developer/neo4j-browser/) can be used to graphically look at the CPG nodes and edges, instead of their Java representations. +In order to get familiar with the graph itself, you can use the subproject [cpg-neo4j](./cpg-neo4j). It uses this library to generate the CPG for a set of user-provided code files. The graph is then persisted to a [Neo4j](https://neo4j.com/) graph database. The advantage this has for the user, is that Neo4j's visualization software [Neo4j Browser](https://neo4j.com/developer/neo4j-browser/) can be used to graphically look at the CPG nodes and edges, instead of their Java representations. + +Please make sure, that the [APOC](https://neo4j.com/labs/apoc/) plugin is enabled on your neo4j server. It is used in mass-creating nodes and relationships. + +For example using docker: +``` +docker run -p 7474:7474 -p 7687:7687 -d -e NEO4J_AUTH=neo4j/password -e NEO4JLABS_PLUGINS='["apoc"]' neo4j:5 +``` ### As Library diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt index a1ef85e5b6c..54fab47fb7f 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt @@ -255,9 +255,7 @@ class Application : Callable { @CommandLine.Option( names = ["--exclusion-patterns"], description = - [ - "Set top level directory of project structure. Default: Largest common path of all source files" - ] + ["Configures an exclusion pattern for files or directories that should not be parsed"] ) private var exclusionPatterns: List = listOf() From 21e2010f37ab1cd95d630235e57c89b8b24c355a Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Sat, 14 Dec 2024 00:22:13 +0100 Subject: [PATCH 22/25] Added test case for BigInteger --- .../aisec/cpg_vis_neo4j/ApplicationTest.kt | 4 ++-- .../aisec/cpg_vis_neo4j/Neo4JTest.kt | 19 +++++++++++++++++-- cpg-neo4j/src/test/resources/very_long.cpp | 1 + 3 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 cpg-neo4j/src/test/resources/very_long.cpp diff --git a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt index bf40cdb2a86..e121a98116d 100644 --- a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt +++ b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt @@ -43,9 +43,9 @@ import kotlin.test.assertNotNull import org.neo4j.ogm.annotation.Relationship import picocli.CommandLine -fun createTranslationResult(): Pair { +fun createTranslationResult(file: String = "client.cpp"): Pair { val topLevel = Paths.get("src").resolve("test").resolve("resources").toAbsolutePath() - val path = topLevel.resolve("client.cpp").toAbsolutePath() + val path = topLevel.resolve(file).toAbsolutePath() val cmd = CommandLine(Application::class.java) cmd.parseArgs(path.toString()) diff --git a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt index a77d2d0269c..91d2d937d38 100644 --- a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt +++ b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt @@ -25,15 +25,17 @@ */ package de.fraunhofer.aisec.cpg_vis_neo4j -import de.fraunhofer.aisec.cpg.graph.functions +import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal +import java.math.BigInteger import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.assertIs import org.junit.jupiter.api.Tag @Tag("integration") class Neo4JTest { @Test - @Throws(InterruptedException::class) fun testPush() { val (application, translationResult) = createTranslationResult() @@ -42,4 +44,17 @@ class Neo4JTest { application.pushToNeo4j(translationResult) } + + @Test + fun testPushVeryLong() { + val (application, translationResult) = createTranslationResult("very_long.cpp") + + assertEquals(1, translationResult.variables.size) + + val lit = translationResult.variables["l"]?.initializer + assertIs>(lit) + assertEquals(BigInteger("10958011617037158669"), lit.value) + + application.pushToNeo4j(translationResult) + } } diff --git a/cpg-neo4j/src/test/resources/very_long.cpp b/cpg-neo4j/src/test/resources/very_long.cpp new file mode 100644 index 00000000000..406c77bf2a8 --- /dev/null +++ b/cpg-neo4j/src/test/resources/very_long.cpp @@ -0,0 +1 @@ +unsigned long long l = 10958011617037158669ull; \ No newline at end of file From 34435a8b327a1c21e832a23528fdbd0a9988f6f0 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Sat, 14 Dec 2024 00:27:44 +0100 Subject: [PATCH 23/25] Adjusted README --- cpg-neo4j/README.md | 47 ++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/cpg-neo4j/README.md b/cpg-neo4j/README.md index b379d89493e..5ef4c510325 100644 --- a/cpg-neo4j/README.md +++ b/cpg-neo4j/README.md @@ -6,6 +6,13 @@ A simple tool to export a *code property graph* to a neo4j database. The application requires Java 17 or higher. +Please make sure, that the [APOC](https://neo4j.com/labs/apoc/) plugin is enabled on your neo4j server. It is used in mass-creating nodes and relationships. + +For example using docker: +``` +docker run -p 7474:7474 -p 7687:7687 -d -e NEO4J_AUTH=neo4j/password -e NEO4JLABS_PLUGINS='["apoc"]' neo4j:5 +``` + ## Build Build (and install) a distribution using Gradle @@ -19,16 +26,19 @@ Please remember to adjust the `gradle.properties` before building the project. ## Usage ``` -./build/install/cpg-neo4j/bin/cpg-neo4j [--infer-nodes] [--load-includes] [--no-default-passes] +./build/install/cpg-neo4j/bin/cpg-neo4j [--infer-nodes] [--load-includes] [--no-default-passes] [--no-neo4j] [--no-purge-db] [--print-benchmark] - [--use-unity-build] [--benchmark-json=] + [--schema-json] [--schema-markdown] [--use-unity-build] + [--benchmark-json=] [--custom-pass-list=] [--export-json=] [--host=] [--includes-file=] + [--max-complexity-cf-dfg=] [--password=] [--port=] [--save-depth=] [--top-level=] - [--user=] ([...] | -S= - [-S=]... | + [--user=] + [--exclusion-patterns=]... ([...] + | -S= [-S=]... | --json-compilation-database= | --list-passes) [...] The paths to analyze. If module support is @@ -37,11 +47,14 @@ Please remember to adjust the `gradle.properties` before building the project. --benchmark-json= Save benchmark results to json file --custom-pass-list= - Add custom list of passes (includes - --no-default-passes) which is passed as a - comma-separated list; give either pass name if - pass is in list, or its FQDN (e.g. + Add custom list of passes (might be used + additional to --no-default-passes) which is + passed as a comma-separated list; give either + pass name if pass is in list, or its FQDN (e.g. --custom-pass-list=DFGPass,CallResolver) + --exclusion-patterns= + Configures an exclusion pattern for files or + directories that should not be parsed --export-json= Export cpg as json --host= Set the host of the neo4j Database (default: @@ -53,6 +66,11 @@ Please remember to adjust the `gradle.properties` before building the project. The path to an optional a JSON compilation database --list-passes Prints the list available passes --load-includes Enable TranslationConfiguration option loadIncludes + --max-complexity-cf-dfg= + Performance optimisation: Limit the + ControlFlowSensitiveDFGPass to functions with a + complexity less than what is specified here. -1 + (default) means no limit is used. --no-default-passes Do not register default passes [used for debugging] --no-neo4j Do not push cpg into neo4j [used for debugging] --no-purge-db Do no purge neo4j database before pushing the cpg @@ -69,6 +87,8 @@ Please remember to adjust the `gradle.properties` before building the project. --save-depth= Performance optimisation: Limit recursion depth form neo4j OGM when leaving the AST. -1 (default) means no limit is used. + --schema-json Print the CPGs nodes and edges that they can have. + --schema-markdown Print the CPGs nodes and edges that they can have. --top-level= Set top level directory of project structure. Default: Largest common path of all source files --use-unity-build Enable unity build mode for C++ (requires @@ -96,13 +116,4 @@ $ build/install/cpg-neo4j/bin/cpg-neo4j --export-json cpg-export.json --no-neo4j ``` To export the cpg from a neo4j database, you can use the neo4j `apoc` plugin. -There it's also possible to export only parts of the graph. - -## Known issues: - -- While importing sufficiently large projects with the parameter --save-depth=-1 - a java.lang.StackOverflowError may occur. - - This error could be solved by increasing the stack size with the JavaVM option: -Xss4m - - Otherwise the depth must be limited (e.g. 3 or 5) - -- While pushing a constant value larger than 2^63 - 1 a java.lang.IllegalArgumentException occurs. +There it's also possible to export only parts of the graph. \ No newline at end of file From 0571d6e4d88856bf5ddcc53be7bf4b4dff979234 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Sat, 14 Dec 2024 00:30:58 +0100 Subject: [PATCH 24/25] Cleanup --- .../de/fraunhofer/aisec/cpg/graph/MermaidPrinter.kt | 2 +- .../fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/MermaidPrinter.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/MermaidPrinter.kt index 1da2d709ce1..65ff85b03b6 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/MermaidPrinter.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/MermaidPrinter.kt @@ -109,7 +109,7 @@ fun > Node.printGraph( private fun Edge.label(): String { val builder = StringBuilder() builder.append("\"") - builder.append(this.labels) + builder.append(this.labels.joinToString(",")) if (this is Dataflow) { var granularity = this.granularity diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt index 5fba6f689a8..e8bb941de20 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/ast/AstEdge.kt @@ -33,11 +33,12 @@ import org.neo4j.ogm.annotation.* /** This property edge describes a parent/child relationship in the Abstract Syntax Tree (AST). */ @RelationshipEntity -open class AstEdge(start: Node, end: T, override var labels: Set = setOf("AST")) : - Edge(start, end) { +open class AstEdge(start: Node, end: T) : Edge(start, end) { init { end.astParent = start } + + override var labels: Set = setOf("AST") } /** Creates an [AstEdges] container starting from this node. */ @@ -53,11 +54,11 @@ fun Node.astEdgesOf( * container). */ fun Node.astOptionalEdgeOf( - onChanged: ((old: AstEdge?, new: AstEdge?) -> Unit)? = null + onChanged: ((old: AstEdge?, new: AstEdge?) -> Unit)? = null, ): EdgeSingletonList> { return EdgeSingletonList( thisRef = this, - init = { start, end -> AstEdge(start, end) }, + init = ::AstEdge, outgoing = true, onChanged = onChanged, of = null @@ -73,7 +74,7 @@ fun Node.astEdgeOf( ): EdgeSingletonList> { return EdgeSingletonList( thisRef = this, - init = { start, end -> AstEdge(start, end) }, + init = ::AstEdge, outgoing = true, onChanged = onChanged, of = of @@ -88,7 +89,7 @@ open class AstEdges>( @Suppress("UNCHECKED_CAST") init: (start: Node, end: NodeType) -> PropertyEdgeType = { start, end -> AstEdge(start, end) as PropertyEdgeType - }, + } ) : EdgeList( thisRef = thisRef, From b00078dc432783071450a6b4d716aa15fe879528 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Sat, 14 Dec 2024 00:44:08 +0100 Subject: [PATCH 25/25] Fixed compile error --- .../aisec/cpg/graph/statements/expressions/CallExpression.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt index 221d6864323..cf9056e16ae 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt @@ -120,7 +120,7 @@ open class CallExpression : /** Adds the specified [expression] with an optional [name] to this call. */ fun addArgument(expression: Expression, name: String? = null) { - val edge = AstEdge(this, expression, labels = setOf("ARGUMENTS", "ARGUMENT")) + val edge = AstEdge(this, expression) edge.name = name argumentEdges.add(edge)