Skip to content

Commit

Permalink
Add exclusion filter for directories (#1828)
Browse files Browse the repository at this point in the history
* Add exclusion filter for directories

* Revert changes from other branch

* Refactor filter

* Add separate props for exclusion by regex and strings

* Overload functions

* Changed method parameter types to vararg

* Added tests

---------

Co-authored-by: Christian Banse <[email protected]>
Co-authored-by: Christian Banse <[email protected]>
  • Loading branch information
3 people authored Nov 29, 2024
1 parent 71fc658 commit 897955d
Show file tree
Hide file tree
Showing 7 changed files with 196 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,10 @@ private constructor(
matchCommentsToNodes: Boolean,
addIncludesToGraph: Boolean,
passConfigurations: Map<KClass<out Pass<*>>, PassConfiguration>,
/** A list of exclusion patterns used to filter files and directories. */
val exclusionPatternsByString: List<String>,
/** A list of exclusion patterns using regular expressions to filter files and directories. */
val exclusionPatternsByRegex: List<Regex>
) {
/** This list contains all languages which we want to translate. */
val languages: List<Language<*>>
Expand Down Expand Up @@ -257,6 +261,8 @@ private constructor(
private var useDefaultPasses = false
private var passConfigurations: MutableMap<KClass<out Pass<*>>, PassConfiguration> =
mutableMapOf()
private val exclusionPatternsByRegex = mutableListOf<Regex>()
private val exclusionPatternsByString = mutableListOf<String>()

fun symbols(symbols: Map<String, String>): Builder {
this.symbols = symbols
Expand Down Expand Up @@ -453,6 +459,32 @@ private constructor(
return this.configurePass(T::class, config)
}

/**
* Adds exclusion patterns using regular expressions for filtering files and directories.
*
* @param patterns Exclusion patterns. Example:
* ```
* exclusionPatterns(Regex(".*test(s)?"))
* ```
*/
fun exclusionPatterns(vararg patterns: Regex): Builder {
exclusionPatternsByRegex.addAll(patterns)
return this
}

/**
* Adds exclusion patterns for filtering files and directories.
*
* @param patterns Exclusion patterns. Example:
* ```
* exclusionPatterns("tests")
* ```
*/
fun exclusionPatterns(vararg patterns: String): Builder {
exclusionPatternsByString.addAll(patterns)
return this
}

/**
* Loads and registers an additional [Language] based on a fully qualified class name (FQN).
*/
Expand Down Expand Up @@ -647,7 +679,9 @@ private constructor(
compilationDatabase,
matchCommentsToNodes,
addIncludesToGraph,
passConfigurations
passConfigurations,
exclusionPatternsByString,
exclusionPatternsByRegex
)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,16 @@ private constructor(
.walkTopDown()
.onEnter { !it.name.startsWith(".") }
.filter { it.isFile && !it.name.startsWith(".") }
.filter {
ctx.config.exclusionPatternsByString.none { pattern ->
it.absolutePath.contains(pattern)
}
}
.filter {
ctx.config.exclusionPatternsByRegex.none { pattern ->
pattern.containsMatchIn(it.absolutePath)
}
}
.toList()
files
} else {
Expand Down
149 changes: 149 additions & 0 deletions cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/ExclusionTest.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
/*
* Copyright (c) 2024, Fraunhofer AISEC. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* $$$$$$\ $$$$$$$\ $$$$$$\
* $$ __$$\ $$ __$$\ $$ __$$\
* $$ / \__|$$ | $$ |$$ / \__|
* $$ | $$$$$$$ |$$ |$$$$\
* $$ | $$ ____/ $$ |\_$$ |
* $$ | $$\ $$ | $$ | $$ |
* \$$$$$ |$$ | \$$$$$ |
* \______/ \__| \______/
*
*/
package de.fraunhofer.aisec.cpg

import de.fraunhofer.aisec.cpg.frontends.Language
import de.fraunhofer.aisec.cpg.frontends.TestLanguage
import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend
import de.fraunhofer.aisec.cpg.graph.Node
import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration
import de.fraunhofer.aisec.cpg.graph.newTranslationUnitDeclaration
import de.fraunhofer.aisec.cpg.graph.types.Type
import de.fraunhofer.aisec.cpg.graph.unknownType
import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation
import java.io.File
import kotlin.reflect.KClass
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNotNull

class TestFileLanguage : TestLanguage() {
override val fileExtensions: List<String>
get() = listOf("file")

override val frontend: KClass<out TestLanguageFrontend>
get() = TestFileLanguageFrontend::class
}

/** Just a test frontend that "reads" a file and returns an empty [TranslationUnitDeclaration]. */
class TestFileLanguageFrontend(
language: Language<TestLanguageFrontend> = TestFileLanguage(),
ctx: TranslationContext =
TranslationContext(
TranslationConfiguration.builder().build(),
ScopeManager(),
TypeManager()
),
) : TestLanguageFrontend("::", language, ctx) {
override fun parse(file: File): TranslationUnitDeclaration {
return newTranslationUnitDeclaration(file.name)
}

override fun typeOf(type: Any): Type {
return unknownType()
}

override fun codeOf(astNode: Any): String? {
return null
}

override fun locationOf(astNode: Any): PhysicalLocation? {
return null
}

override fun setComment(node: Node, astNode: Any) {}
}

class ExclusionTest {
@Test
fun testExclusionPatternStringDirectory() {
val topLevel = File("src/test/resources/exclusion")
val result =
TranslationManager.builder()
.config(
TranslationConfiguration.builder()
.topLevel(topLevel)
.sourceLocations(topLevel)
.defaultPasses()
.exclusionPatterns("tests")
.registerLanguage<TestFileLanguage>()
.build()
)
.build()
.analyze()
.get()

val tus = result.translationUnits
assertNotNull(tus)
assertEquals(1, tus.size)
}

@Test
fun testExclusionPatternStringFile() {
val topLevel = File("src/test/resources/exclusion")
val result =
TranslationManager.builder()
.config(
TranslationConfiguration.builder()
.topLevel(topLevel)
.sourceLocations(topLevel)
.defaultPasses()
.exclusionPatterns("test.file")
.registerLanguage<TestFileLanguage>()
.build()
)
.build()
.analyze()
.get()

val tus = result.translationUnits
assertNotNull(tus)
assertEquals(1, tus.size)
}

@Test
fun testExclusionPatternRegex() {
val topLevel = File("src/test/resources/exclusion")
val result =
TranslationManager.builder()
.config(
TranslationConfiguration.builder()
.topLevel(topLevel)
.sourceLocations(topLevel)
.defaultPasses()
.exclusionPatterns("""(.*)est.file""".toRegex())
.registerLanguage<TestFileLanguage>()
.build()
)
.build()
.analyze()
.get()

val tus = result.translationUnits
assertNotNull(tus)
assertEquals(1, tus.size)
}
}
Empty file.
Empty file.
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,6 @@ open class TestLanguage(final override var namespaceDelimiter: String = "::") :
"double" to FloatingPointType("double", 64, this, NumericType.Modifier.SIGNED),
"string" to StringType("string", this),
)

override fun newFrontend(ctx: TranslationContext): TestLanguageFrontend {
return TestLanguageFrontend(language = this, ctx = ctx)
}
}

class StructTestLanguage(namespaceDelimiter: String = "::") :
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -113,9 +113,8 @@ fun analyze(
): TranslationResult {
val files =
Files.walk(topLevel, Int.MAX_VALUE)
.map(Path::toFile)
.filter { it.isFile }
.filter { it.name.endsWith(fileExtension!!) }
.map { it.toFile() }
.filter { it.isFile && (fileExtension == null || it.name.endsWith(fileExtension)) }
.sorted()
.collect(Collectors.toList())
return analyze(files, topLevel, usePasses, configModifier)
Expand Down

0 comments on commit 897955d

Please sign in to comment.