From 866d2b3378891a89f2ca92ac041c4b2cc3ecd128 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Sat, 14 Dec 2024 00:12:59 +0100 Subject: [PATCH] Using Neo4J converter annotation for now --- .../aisec/cpg/TranslationConfiguration.kt | 2 + .../aisec/cpg/TranslationContext.kt | 2 + .../fraunhofer/aisec/cpg/TranslationResult.kt | 2 + .../aisec/cpg/helpers/MeasurementHolder.kt | 3 + .../aisec/cpg/persistence/Common.kt | 129 +++++++++++------- .../aisec/cpg/persistence/TestCommon.kt | 20 ++- 6 files changed, 104 insertions(+), 54 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt index 3c4b1906f37..bb13e63375e 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt @@ -36,6 +36,7 @@ import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.passes.* import de.fraunhofer.aisec.cpg.passes.configuration.* import de.fraunhofer.aisec.cpg.passes.inference.DFGFunctionSummaries +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import java.io.File import java.nio.file.Path import java.util.* @@ -51,6 +52,7 @@ import org.slf4j.LoggerFactory * The configuration for the [TranslationManager] holds all information that is used during the * translation. */ +@DoNotPersist class TranslationConfiguration private constructor( /** Definition of additional symbols, mostly useful for C++. */ diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationContext.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationContext.kt index 3d7636ee127..1524fe7050e 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationContext.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationContext.kt @@ -26,11 +26,13 @@ package de.fraunhofer.aisec.cpg import de.fraunhofer.aisec.cpg.graph.Component +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist /** * The translation context holds all necessary managers and configurations needed during the * translation process. */ +@DoNotPersist class TranslationContext( /** The configuration for this translation. */ val config: TranslationConfiguration, diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt index 05ace1a4dc8..7b4b3ddfb6a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt @@ -35,6 +35,7 @@ import de.fraunhofer.aisec.cpg.graph.edges.unwrapping import de.fraunhofer.aisec.cpg.helpers.MeasurementHolder import de.fraunhofer.aisec.cpg.helpers.StatisticsHolder import de.fraunhofer.aisec.cpg.passes.Pass +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import java.util.* import java.util.concurrent.ConcurrentHashMap import org.neo4j.ogm.annotation.Relationship @@ -92,6 +93,7 @@ class TranslationResult( * @return the list of all translation units. */ @Deprecated(message = "translation units of individual components should be accessed instead") + @DoNotPersist val translationUnits: List get() { if (components.size == 1) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/MeasurementHolder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/MeasurementHolder.kt index f16b284cdc8..2072ccae8f9 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/MeasurementHolder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/MeasurementHolder.kt @@ -28,6 +28,7 @@ package de.fraunhofer.aisec.cpg.helpers import com.fasterxml.jackson.databind.SerializationFeature import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import de.fraunhofer.aisec.cpg.TranslationConfiguration +import de.fraunhofer.aisec.cpg.persistence.DoNotPersist import java.io.File import java.nio.file.Path import java.time.Duration @@ -36,6 +37,7 @@ import java.util.* import org.slf4j.Logger import org.slf4j.LoggerFactory +@DoNotPersist class BenchmarkResults(val entries: List>) { val json: String @@ -175,6 +177,7 @@ constructor( } } +@DoNotPersist /** Represents some kind of measurements, e.g., on the performance or problems. */ open class MeasurementHolder @JvmOverloads diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt index d2d9a64e2ae..27a25541ae6 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/persistence/Common.kt @@ -25,27 +25,19 @@ */ package de.fraunhofer.aisec.cpg.persistence -import de.fraunhofer.aisec.cpg.TranslationConfiguration -import de.fraunhofer.aisec.cpg.TranslationContext import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.Persistable import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeCollection import de.fraunhofer.aisec.cpg.graph.edges.collections.EdgeList -import de.fraunhofer.aisec.cpg.graph.edges.flows.DependenceType -import de.fraunhofer.aisec.cpg.graph.edges.flows.Granularity -import de.fraunhofer.aisec.cpg.helpers.BenchmarkResults -import de.fraunhofer.aisec.cpg.helpers.neo4j.DataflowGranularityConverter -import de.fraunhofer.aisec.cpg.helpers.neo4j.LocationConverter import de.fraunhofer.aisec.cpg.helpers.neo4j.NameConverter -import de.fraunhofer.aisec.cpg.helpers.neo4j.SimpleNameConverter -import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation import java.math.BigInteger import kotlin.reflect.KClass import kotlin.reflect.KProperty1 import kotlin.reflect.KTypeProjection import kotlin.reflect.KVariance import kotlin.reflect.KVisibility +import kotlin.reflect.full.createInstance import kotlin.reflect.full.createType import kotlin.reflect.full.hasAnnotation import kotlin.reflect.full.isSubtypeOf @@ -57,6 +49,8 @@ import kotlin.reflect.jvm.javaType import kotlin.uuid.Uuid import org.neo4j.ogm.annotation.Relationship import org.neo4j.ogm.annotation.Relationship.Direction.INCOMING +import org.neo4j.ogm.annotation.typeconversion.Convert +import org.neo4j.ogm.typeconversion.AttributeConverter import org.neo4j.ogm.typeconversion.CompositeAttributeConverter /** @@ -90,22 +84,6 @@ val schemaRelationshipCache: MutableMap, Map>> = mutableMapOf() -/** - * A set containing predefined property types represented as Kotlin type objects that can be used as - * properties in [schemaProperties]. - */ -val propertyTypes = - setOf( - String::class.createType(), - Int::class.createType(), - Long::class.createType(), - Boolean::class.createType(), - Name::class.createType(), - Uuid::class.createType(), - Granularity::class.createType(), - DependenceType::class.createType(), - ) - /** * Returns the [Persistable]'s properties. This DOES NOT include relationships, but only properties * directly attached to the node/edge. @@ -119,7 +97,7 @@ fun Persistable.properties(): Map { continue } - value.convert(entry.key, properties) + value.convert(entry, properties) } return properties @@ -127,19 +105,31 @@ fun Persistable.properties(): Map { /** * Runs any conversions that are necessary by [CompositeAttributeConverter] and - * [org.neo4j.ogm.typeconversion.AttributeConverter]. Since both of these classes are Neo4J OGM - * classes, we need to find new base types at some point. + * [AttributeConverter]. Since both of these classes are Neo4J OGM classes, we need to find new base + * types at some point. */ -fun Any.convert(originalKey: String, properties: MutableMap) { - // TODO: generalize conversions - if (this is Name && originalKey == "name") { +fun Any.convert( + entry: Map.Entry>, + properties: MutableMap +) { + val originalKey = entry.key + + val annotation = entry.value.javaField?.getAnnotation(Convert::class.java) + @Suppress("UNCHECKED_CAST") + if (annotation != null) { + val converter = annotation.value.createInstance() + if (converter is CompositeAttributeConverter<*>) { + properties += (converter as CompositeAttributeConverter).toGraphProperties(this) + } else if (converter is AttributeConverter<*, *>) { + properties.put( + originalKey, + (converter as AttributeConverter).toGraphProperty(this) + ) + } + } else if (this is Name && originalKey == "name") { + // needs to be extra because of the way annotations work, this will be re-designed once OGM + // is completely gone properties += NameConverter().toGraphProperties(this) - } else if (this is Name) { - properties.put(originalKey, SimpleNameConverter().toGraphProperty(this)) - } else if (this is PhysicalLocation) { - properties += LocationConverter().toGraphProperties(this) - } else if (this is Granularity) { - properties += DataflowGranularityConverter().toGraphProperties(this) } else if (this is Enum<*>) { properties.put(originalKey, this.name) } else if (this is Uuid) { @@ -184,15 +174,7 @@ val KClass<*>.labels: Set return labels } -/** A list of specific types that are intended to be ignored for persistence. */ -internal val ignoredTypes = - listOf( - TranslationContext::class.createType(), - TranslationConfiguration::class.createType(), - BenchmarkResults::class.createType(), - KClass::class.createType(listOf(KTypeProjection.STAR)), - ) - +internal val kClassType = KClass::class.createType(listOf(KTypeProjection.STAR)) internal val nodeType = Node::class.createType() internal val collectionType = Collection::class.createType(listOf(KTypeProjection.STAR)) internal val collectionOfNodeType = @@ -206,9 +188,8 @@ internal val mapType = Map::class.createType(listOf(KTypeProjection.STAR, KTypeP * [Persistable]. * * This property computes a map that associates property names (as strings) to their corresponding - * [KProperty1] objects, which represent the properties defined in the class. Only properties whose - * return types are included in a predefined set of supported property types ([propertyTypes]) are - * included in the map. + * [KProperty1] objects, which represent the properties defined in the class. Only properties for + * which [isSimpleProperty] returns true, are included. * * The computed map is cached to optimize subsequent lookups for properties of the same class. */ @@ -227,6 +208,25 @@ val KClass.schemaProperties: Map.schemaRelationships: Map> get() { // Check, if we already computed the relationship for this node's class @@ -249,10 +249,11 @@ val KClass.schemaRelationships: Map): Boolean val returnType = property.returnType.withNullability(false) return when { - property.hasAnnotation() -> false property.visibility == KVisibility.PRIVATE -> false - ignoredTypes.any { returnType.isSubtypeOf(it) } -> false + property.hasAnnotation() -> false + returnType.hasAnnotation() -> false + (returnType.javaType as? Class<*>)?.getAnnotation(DoNotPersist::class.java) != null -> false + returnType.isSubtypeOf(kClassType) -> false returnType.isSubtypeOf(collectionType) -> false returnType.isSubtypeOf(mapType) -> false returnType.isSubtypeOf(nodeType) -> false @@ -306,6 +309,17 @@ private fun isRelationship(property: KProperty1): Boolean { } } +/** + * Retrieves the relational name associated with a property in the context of the raph schema. + * + * The `relationshipName` is determined based on the following rules: + * - If the property is annotated with the `@Relationship` annotation, the value of the annotation + * is used as the relationship name, provided it is non-null and not an empty string. + * - If the property name ends with "Edge", this suffix is removed. This adjustment is made to + * account for cases where two variables represent an edge, one named with "Edge" and another as + * the delegate without the suffix. The desired name is the one without "Edge". + * - The resulting name is converted to UPPER_SNAKE_CASE for standardization. + */ val KProperty1.relationshipName: String get() { // If we have a (legacy) Neo4J annotation for our relationship, we take this one @@ -323,6 +337,15 @@ val KProperty1.relationshipName: String return this.name.substringBeforeLast("Edge").toUpperSnakeCase() } +/** + * Converts the current string to UPPER_SNAKE_CASE format. + * + * Each word boundary in camelCase or PascalCase naming convention is replaced with an underscore, + * and all characters are converted to uppercase. This is commonly used for representing constants + * or environment-style variable names. + * + * @return A string converted to UPPER_SNAKE_CASE. + */ fun String.toUpperSnakeCase(): String { val pattern = "(?<=.)[A-Z]".toRegex() return this.replace(pattern, "_$0").uppercase() diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/persistence/TestCommon.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/persistence/TestCommon.kt index c53fc609a10..8d68203300b 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/persistence/TestCommon.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/persistence/TestCommon.kt @@ -25,6 +25,7 @@ */ package de.fraunhofer.aisec.cpg.persistence +import de.fraunhofer.aisec.cpg.TranslationResult import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import kotlin.test.Test import kotlin.test.assertEquals @@ -54,7 +55,7 @@ class TestCommon { @Test fun testSchemaRelationships() { - val relationships = FunctionDeclaration::class.schemaRelationships + var relationships = FunctionDeclaration::class.schemaRelationships assertEquals( listOf( "ANNOTATIONS", @@ -78,5 +79,22 @@ class TestCommon { ), relationships.keys.sorted() ) + + relationships = TranslationResult::class.schemaRelationships + assertEquals( + listOf( + "ADDITIONAL_NODES", + "ANNOTATIONS", + "AST", + "CDG", + "COMPONENTS", + "DFG", + "EOG", + "LANGUAGE", + "PDG", + "SCOPE", + ), + relationships.keys.sorted() + ) } }