Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Persisting to Neo4J without OGM #1893

Merged
merged 26 commits into from
Dec 13, 2024
Merged
Changes from 1 commit
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Adding labels to ast edges
oxisto committed Dec 11, 2024
commit 30ee8e408cb74e8fad2123f1452987ea71a5d882
Original file line number Diff line number Diff line change
@@ -54,7 +54,7 @@ class TranslationResult(
var finalCtx: TranslationContext,
) : Node(), StatisticsHolder {

@Relationship("COMPONENTS") val componentEdges = astEdgesOf<Component>()
@Relationship("COMPONENTS") val componentEdges = astEdgesOf<Component>(label = "COMPONENTS")
/**
* Entry points to the CPG: "SoftwareComponent" refer to programs, application, other "bundles"
* of software.
Original file line number Diff line number Diff line change
@@ -323,7 +323,7 @@ abstract class Language<T : LanguageFrontend<*, *>> : Node() {
// matches
val source = result.source
if (this is HasTemplates && source is CallExpression) {
source.templateArgumentEdges = TemplateArguments(source)
source.templateArgumentEdges = TemplateArguments(source, label = "TEMPLATE_ARGUMENTS")
val (ok, candidates) =
this.handleTemplateFunctionCalls(
null,
Original file line number Diff line number Diff line change
@@ -43,7 +43,7 @@ import org.neo4j.ogm.annotation.Transient
*/
open class Component : Node() {
@Relationship("TRANSLATION_UNITS")
val translationUnitEdges = astEdgesOf<TranslationUnitDeclaration>()
val translationUnitEdges = astEdgesOf<TranslationUnitDeclaration>(label = "TRANSLATION_UNITS")
/** All translation units belonging to this application. */
val translationUnits by unwrapping(Component::translationUnitEdges)

Original file line number Diff line number Diff line change
@@ -48,6 +48,7 @@ import de.fraunhofer.aisec.cpg.passes.*
import de.fraunhofer.aisec.cpg.processing.IVisitable
import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation
import java.util.*
import kotlin.uuid.Uuid
import org.apache.commons.lang3.builder.ToStringBuilder
import org.apache.commons.lang3.builder.ToStringStyle
import org.neo4j.ogm.annotation.*
@@ -254,11 +255,14 @@ abstract class Node :
/** Required field for object graph mapping. It contains the node id. */
@Id @GeneratedValue var id: Long? = null

/** Will replace [id] */
var graphId: Uuid? = null

/** Index of the argument if this node is used in a function call or parameter list. */
var argumentIndex = 0

/** List of annotations associated with that node. */
@Relationship("ANNOTATIONS") var annotationEdges = astEdgesOf<Annotation>()
@Relationship("ANNOTATIONS") var annotationEdges = astEdgesOf<Annotation>(label = "ANNOTATIONS")
var annotations by unwrapping(Node::annotationEdges)

/**
Original file line number Diff line number Diff line change
@@ -36,6 +36,7 @@ import org.neo4j.ogm.annotation.Relationship
* explicit initializer value.
*/
class EnumConstantDeclaration : ValueDeclaration(), HasInitializer {
@Relationship("INITIALIZER") var initializerEdge = astOptionalEdgeOf<Expression>()
@Relationship("INITIALIZER")
var initializerEdge = astOptionalEdgeOf<Expression>(label = "INITIALIZER")
override var initializer by unwrapping(EnumConstantDeclaration::initializerEdge)
}
Original file line number Diff line number Diff line change
@@ -32,7 +32,7 @@ import org.neo4j.ogm.annotation.Relationship

class EnumDeclaration : RecordDeclaration() {
@Relationship(value = "ENTRIES", direction = Relationship.Direction.OUTGOING)
var entryEdges = astEdgesOf<EnumConstantDeclaration>()
var entryEdges = astEdgesOf<EnumConstantDeclaration>(label = "ENTRIES")
var entries by unwrapping(EnumDeclaration::entryEdges)

override fun toString(): String {
Original file line number Diff line number Diff line change
@@ -40,13 +40,13 @@ import org.neo4j.ogm.annotation.Relationship

/** Represents the declaration or definition of a function. */
open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder, EOGStarterHolder {
@Relationship("BODY") var bodyEdge = astOptionalEdgeOf<Statement>()
@Relationship("BODY") var bodyEdge = astOptionalEdgeOf<Statement>(label = "BODY")
/** The function body. Usually a [Block]. */
var body by unwrapping(FunctionDeclaration::bodyEdge)

/** The list of function parameters. */
@Relationship(value = "PARAMETERS", direction = Relationship.Direction.OUTGOING)
var parameterEdges = astEdgesOf<ParameterDeclaration>()
var parameterEdges = astEdgesOf<ParameterDeclaration>(label = "PARAMETERS")
/** Virtual property for accessing [parameterEdges] without property edges. */
var parameters by unwrapping(FunctionDeclaration::parameterEdges)

Original file line number Diff line number Diff line change
@@ -39,7 +39,7 @@ class FunctionTemplateDeclaration : TemplateDeclaration() {
* expansion pass for each instantiation of the FunctionTemplate there will be a realization
*/
@Relationship(value = "REALIZATION", direction = Relationship.Direction.OUTGOING)
val realizationEdges = astEdgesOf<FunctionDeclaration>()
val realizationEdges = astEdgesOf<FunctionDeclaration>(label = "REALIZATION")
val realization by unwrapping(FunctionTemplateDeclaration::realizationEdges)

override val realizations: List<Declaration>
Original file line number Diff line number Diff line change
@@ -35,11 +35,11 @@ import org.neo4j.ogm.annotation.Relationship
/** This declaration represents either an include or an import, depending on the language. */
class IncludeDeclaration : Declaration() {
@Relationship(value = "INCLUDES", direction = Relationship.Direction.OUTGOING)
val includeEdges = astEdgesOf<IncludeDeclaration>()
val includeEdges = astEdgesOf<IncludeDeclaration>(label = "INCLUDES")
val includes by unwrapping(IncludeDeclaration::includeEdges)

@Relationship(value = "PROBLEMS", direction = Relationship.Direction.OUTGOING)
val problemEdges = astEdgesOf<ProblemDeclaration>()
val problemEdges = astEdgesOf<ProblemDeclaration>(label = "PROBLEMS")
val problems by unwrapping(IncludeDeclaration::problemEdges)

/**
Original file line number Diff line number Diff line change
@@ -43,7 +43,8 @@ open class MethodDeclaration : FunctionDeclaration() {
*/
open var recordDeclaration: RecordDeclaration? = null

@Relationship("RECEIVER") var receiverEdge = astOptionalEdgeOf<VariableDeclaration>()
@Relationship("RECEIVER")
var receiverEdge = astOptionalEdgeOf<VariableDeclaration>(label = "RECEIVER")
/**
* The receiver variable of this method. In most cases, this variable is called `this`, but in
* some languages, it is `self` (e.g. in Rust or Python) or can be freely named (e.g. in
Original file line number Diff line number Diff line change
@@ -48,12 +48,12 @@ class NamespaceDeclaration : Declaration(), DeclarationHolder, StatementHolder,
* Edges to nested namespaces, records, functions, fields etc. contained in the current
* namespace.
*/
val declarationEdges = astEdgesOf<Declaration>()
val declarationEdges = astEdgesOf<Declaration>(label = "DECLARATIONS")
override val declarations by unwrapping(NamespaceDeclaration::declarationEdges)

/** The list of statements. */
@Relationship(value = "STATEMENTS", direction = Relationship.Direction.OUTGOING)
override var statementEdges = astEdgesOf<Statement>()
override var statementEdges = astEdgesOf<Statement>(label = "STATEMENTS")

/**
* In some languages, there is a relationship between paths / directories and the package
Original file line number Diff line number Diff line change
@@ -37,7 +37,7 @@ class ParameterDeclaration : ValueDeclaration(), HasDefault<Expression?> {
var isVariadic = false

@Relationship(value = "DEFAULT", direction = Relationship.Direction.OUTGOING)
var defaultValueEdge = astOptionalEdgeOf<Expression>()
var defaultValueEdge = astOptionalEdgeOf<Expression>(label = "DEFAULT")
private var defaultValue by unwrapping(ParameterDeclaration::defaultValueEdge)

var modifiers: List<String> = mutableListOf()
Original file line number Diff line number Diff line change
@@ -44,28 +44,28 @@ open class RecordDeclaration :
var kind: String? = null

@Relationship(value = "FIELDS", direction = Relationship.Direction.OUTGOING)
var fieldEdges = astEdgesOf<FieldDeclaration>()
var fieldEdges = astEdgesOf<FieldDeclaration>(label = "FIELDS")
var fields by unwrapping(RecordDeclaration::fieldEdges)

@Relationship(value = "METHODS", direction = Relationship.Direction.OUTGOING)
var methodEdges = astEdgesOf<MethodDeclaration>()
var methodEdges = astEdgesOf<MethodDeclaration>(label = "METHODS")
var methods by unwrapping(RecordDeclaration::methodEdges)

@Relationship(value = "CONSTRUCTORS", direction = Relationship.Direction.OUTGOING)
var constructorEdges = astEdgesOf<ConstructorDeclaration>()
var constructorEdges = astEdgesOf<ConstructorDeclaration>(label = "CONSTRUCTORS")
var constructors by unwrapping(RecordDeclaration::constructorEdges)

@Relationship(value = "RECORDS", direction = Relationship.Direction.OUTGOING)
var recordEdges = astEdgesOf<RecordDeclaration>()
var recordEdges = astEdgesOf<RecordDeclaration>(label = "RECORDS")
var records by unwrapping(RecordDeclaration::recordEdges)

@Relationship(value = "TEMPLATES", direction = Relationship.Direction.OUTGOING)
var templateEdges = astEdgesOf<TemplateDeclaration>()
var templateEdges = astEdgesOf<TemplateDeclaration>(label = "TEMPLATES")
var templates by unwrapping(RecordDeclaration::templateEdges)

/** The list of statements. */
@Relationship(value = "STATEMENTS", direction = Relationship.Direction.OUTGOING)
override var statementEdges = astEdgesOf<Statement>()
override var statementEdges = astEdgesOf<Statement>(label = "STATEMENTS")
override var statements by unwrapping(RecordDeclaration::statementEdges)

@Transient var superClasses: MutableList<Type> = ArrayList()
Original file line number Diff line number Diff line change
@@ -39,7 +39,7 @@ class RecordTemplateDeclaration : TemplateDeclaration() {
* expansion pass for each instantiation of the ClassTemplate there will be a realization
*/
@Relationship(value = "REALIZATION", direction = Relationship.Direction.OUTGOING)
val realizationEdges = astEdgesOf<RecordDeclaration>()
val realizationEdges = astEdgesOf<RecordDeclaration>(label = "REALIZATION")
override val realizations by unwrapping(RecordTemplateDeclaration::realizationEdges)

override fun addDeclaration(declaration: Declaration) {
Original file line number Diff line number Diff line change
@@ -51,7 +51,7 @@ abstract class TemplateDeclaration : Declaration(), DeclarationHolder {

/** Parameters the Template requires for instantiation */
@Relationship(value = "PARAMETERS", direction = Relationship.Direction.OUTGOING)
var parameterEdges = astEdgesOf<Declaration>()
var parameterEdges = astEdgesOf<Declaration>(label = "PARAMETERS")
val parameters by unwrapping(TemplateDeclaration::parameterEdges)

val parametersWithDefaults: List<Declaration>
Original file line number Diff line number Diff line change
@@ -39,22 +39,22 @@ class TranslationUnitDeclaration :
Declaration(), DeclarationHolder, StatementHolder, EOGStarterHolder {
/** A list of declarations within this unit. */
@Relationship(value = "DECLARATIONS", direction = Relationship.Direction.OUTGOING)
val declarationEdges = astEdgesOf<Declaration>()
val declarationEdges = astEdgesOf<Declaration>(label = "DECLARATIONS")
override val declarations by unwrapping(TranslationUnitDeclaration::declarationEdges)

/** A list of includes within this unit. */
@Relationship(value = "INCLUDES", direction = Relationship.Direction.OUTGOING)
val includeEdges = astEdgesOf<IncludeDeclaration>()
val includeEdges = astEdgesOf<IncludeDeclaration>(label = "INCLUDES")
val includes by unwrapping(TranslationUnitDeclaration::includeEdges)

/** A list of namespaces within this unit. */
@Relationship(value = "NAMESPACES", direction = Relationship.Direction.OUTGOING)
val namespaceEdges = astEdgesOf<NamespaceDeclaration>()
val namespaceEdges = astEdgesOf<NamespaceDeclaration>(label = "NAMESPACES")
val namespaces by unwrapping(TranslationUnitDeclaration::namespaceEdges)

/** The list of statements. */
@Relationship(value = "STATEMENTS", direction = Relationship.Direction.OUTGOING)
override var statementEdges = astEdgesOf<Statement>()
override var statementEdges = astEdgesOf<Statement>(label = "STATEMENTS")
override var statements by unwrapping(TranslationUnitDeclaration::statementEdges)

override fun addDeclaration(declaration: Declaration) {
Original file line number Diff line number Diff line change
@@ -65,6 +65,7 @@ class TupleDeclaration : VariableDeclaration() {
/** The list of elements in this tuple. */
var elementEdges =
astEdgesOf<VariableDeclaration>(
label = "ELEMENTS",
onAdd = { registerTypeObserver(it.end) },
onRemove = { unregisterTypeObserver(it.end) }
)
Original file line number Diff line number Diff line change
@@ -35,7 +35,7 @@ import org.neo4j.ogm.annotation.Relationship
/** A declaration of a type template parameter */
class TypeParameterDeclaration : ValueDeclaration(), HasDefault<Type?> {
@Relationship(value = "DEFAULT", direction = Relationship.Direction.OUTGOING)
var defaultEdge = astOptionalEdgeOf<Type>()
var defaultEdge = astOptionalEdgeOf<Type>(label = "DEFAULT")
/** TemplateParameters can define a default for the type parameter. */
override var default by unwrapping(TypeParameterDeclaration::defaultEdge)

Original file line number Diff line number Diff line change
@@ -48,7 +48,7 @@ open class VariableDeclaration : ValueDeclaration(), HasInitializer, HasType.Typ
* the [ConstructExpression] is created.
*/
@Relationship(value = "TEMPLATE_PARAMETERS", direction = Relationship.Direction.OUTGOING)
var templateParameterEdges = astEdgesOf<Node>()
var templateParameterEdges = astEdgesOf<Node>(label = "TEMPLATE_PARAMETERS")
var templateParameters by unwrapping(VariableDeclaration::templateParameterEdges)

/** Determines if this is a global variable. */
@@ -69,13 +69,14 @@ open class VariableDeclaration : ValueDeclaration(), HasInitializer, HasType.Typ
@Relationship("INITIALIZER")
var initializerEdge =
astOptionalEdgeOf<Expression>(
label = "INITIALIZER",
onChanged = { old, new ->
val value = new?.end
exchangeTypeObserver(old, new)
if (value is Reference) {
value.resolutionHelper = this
}
}
},
)
/** The (optional) initializer of the declaration. */
override var initializer by unwrapping(VariableDeclaration::initializerEdge)
Original file line number Diff line number Diff line change
@@ -71,7 +71,7 @@ abstract class Edge<NodeType : Node> : Persistable, Cloneable {
end = edge.end
}

@Transient open val label: String = "EDGE"
@Transient open var label: String = "EDGE"

/**
* The index of this node, if it is stored in an
Original file line number Diff line number Diff line change
@@ -33,30 +33,33 @@ import org.neo4j.ogm.annotation.*

/** This property edge describes a parent/child relationship in the Abstract Syntax Tree (AST). */
@RelationshipEntity
open class AstEdge<T : Node>(start: Node, end: T) : Edge<T>(start, end) {
open class AstEdge<T : Node>(start: Node, end: T, label: String = "AST") : Edge<T>(start, end) {
init {
end.astParent = start
this.label = label
}
}

/** Creates an [AstEdges] container starting from this node. */
fun <NodeType : Node> Node.astEdgesOf(
label: String = "AST",
onAdd: ((AstEdge<NodeType>) -> Unit)? = null,
onRemove: ((AstEdge<NodeType>) -> Unit)? = null,
): AstEdges<NodeType, AstEdge<NodeType>> {
return AstEdges(thisRef = this, onAdd = onAdd, onRemove = onRemove)
return AstEdges(thisRef = this, label = label, onAdd = onAdd, onRemove = onRemove)
}

/**
* Creates a single optional [AstEdge] starting from this node (wrapped in a [EdgeSingletonList]
* container).
*/
fun <NodeType : Node> Node.astOptionalEdgeOf(
onChanged: ((old: AstEdge<NodeType>?, new: AstEdge<NodeType>?) -> Unit)? = null,
label: String = "AST",
onChanged: ((old: AstEdge<NodeType>?, new: AstEdge<NodeType>?) -> Unit)? = null
): EdgeSingletonList<NodeType, NodeType?, AstEdge<NodeType>> {
return EdgeSingletonList(
thisRef = this,
init = ::AstEdge,
init = { start, end -> AstEdge(start, end, label = label) },
outgoing = true,
onChanged = onChanged,
of = null
@@ -68,11 +71,12 @@ fun <NodeType : Node> Node.astOptionalEdgeOf(
*/
fun <NodeType : Node> Node.astEdgeOf(
of: NodeType,
label: String,
onChanged: ((old: AstEdge<NodeType>?, new: AstEdge<NodeType>?) -> Unit)? = null,
): EdgeSingletonList<NodeType, NodeType, AstEdge<NodeType>> {
return EdgeSingletonList(
thisRef = this,
init = ::AstEdge,
init = { start, end -> AstEdge(start, end, label = label) },
outgoing = true,
onChanged = onChanged,
of = of
@@ -82,12 +86,13 @@ fun <NodeType : Node> Node.astEdgeOf(
/** This property edge list describes elements that are AST children of a node. */
open class AstEdges<NodeType : Node, PropertyEdgeType : AstEdge<NodeType>>(
thisRef: Node,
label: String = "AST",
onAdd: ((PropertyEdgeType) -> Unit)? = null,
onRemove: ((PropertyEdgeType) -> Unit)? = null,
@Suppress("UNCHECKED_CAST")
init: (start: Node, end: NodeType) -> PropertyEdgeType = { start, end ->
AstEdge(start, end) as PropertyEdgeType
}
AstEdge(start, end, label = label) as PropertyEdgeType
},
) :
EdgeList<NodeType, PropertyEdgeType>(
thisRef = thisRef,
Original file line number Diff line number Diff line change
@@ -33,9 +33,14 @@ import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression
class TemplateArgument<NodeType : Node>(
start: Node,
end: NodeType,
label: String,
var instantiation: TemplateInitialization? = TemplateInitialization.EXPLICIT,
) : AstEdge<NodeType>(start, end)
) : AstEdge<NodeType>(start, end, label)

/** A container for [TemplateArgument] edges. */
class TemplateArguments<NodeType : Node>(thisRef: Node) :
AstEdges<NodeType, TemplateArgument<NodeType>>(thisRef, init = ::TemplateArgument)
class TemplateArguments<NodeType : Node>(thisRef: Node, label: String) :
AstEdges<NodeType, TemplateArgument<NodeType>>(
thisRef,
label,
init = { start, end -> TemplateArgument(start, end, label = label) }
)
Original file line number Diff line number Diff line change
@@ -49,6 +49,8 @@ class ControlDependence(
dependence = DependenceType.CONTROL
}

override var label: String = "CDG"

override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other !is ControlDependence) return false
Original file line number Diff line number Diff line change
@@ -93,7 +93,7 @@ open class Dataflow(
@JsonIgnore
var granularity: Granularity = default()
) : Edge<Node>(start, end) {
override val label: String = "DFG"
override var label: String = "DFG"

override fun equals(other: Any?): Boolean {
if (this === other) return true
@@ -134,7 +134,7 @@ class ContextSensitiveDataflow(
val callingContext: CallingContext
) : Dataflow(start, end, granularity) {

override val label: String = "DFG"
override var label: String = "DFG"

override fun equals(other: Any?): Boolean {
if (this === other) return true
Loading