Skip to content

Commit

Permalink
Merge branch 'main' into cleanup-scopes-part1
Browse files Browse the repository at this point in the history
  • Loading branch information
oxisto authored Dec 2, 2024
2 parents 8d0858d + d40f4eb commit 150ad6a
Show file tree
Hide file tree
Showing 11 changed files with 260 additions and 51 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ on:

jobs:
build:
runs-on: self-hosted
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- run: |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ class TranslatePlugin : Plugin {
" .optionalLanguage(\"de.fraunhofer.aisec.cpg.frontends.golang.GoLanguage\")" +
" .optionalLanguage(\"de.fraunhofer.aisec.cpg.frontends.typescript.TypeScriptLanguage\")" +
" .optionalLanguage(\"de.fraunhofer.aisec.cpg.frontends.ruby.RubyLanguage\")" +
" .optionalLanguage(\"de.fraunhofer.aisec.cpg.frontends.ini.IniFileLanguage\")" +
" .defaultPasses()\n" +
" .useParallelPasses(false)\n" +
" .configurePass<de.fraunhofer.aisec.cpg.passes.ControlFlowSensitiveDFGPass>(\n" +
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
* \______/ \__| \______/
*
*/
package de.fraunhofer.aisec.cpg.frontend.configfiles
package de.fraunhofer.aisec.cpg.frontends.ini

import de.fraunhofer.aisec.cpg.TranslationContext
import de.fraunhofer.aisec.cpg.frontends.Handler
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
* \______/ \__| \______/
*
*/
package de.fraunhofer.aisec.cpg.frontend.configfiles
package de.fraunhofer.aisec.cpg.frontends.ini

import de.fraunhofer.aisec.cpg.frontends.Language
import de.fraunhofer.aisec.cpg.graph.types.StringType
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
* \______/ \__| \______/
*
*/
package de.fraunhofer.aisec.cpg.frontend.configfiles
package de.fraunhofer.aisec.cpg.frontends.ini

import de.fraunhofer.aisec.cpg.graph.declarations.FieldDeclaration
import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,25 +29,11 @@ import de.fraunhofer.aisec.cpg.graph.*
import de.fraunhofer.aisec.cpg.graph.declarations.ImportDeclaration
import de.fraunhofer.aisec.cpg.graph.declarations.MethodDeclaration
import de.fraunhofer.aisec.cpg.graph.statements.expressions.*
import de.fraunhofer.aisec.cpg.graph.statements.expressions.CollectionComprehension
import jep.python.PyObject

class ExpressionHandler(frontend: PythonLanguageFrontend) :
PythonHandler<Expression, Python.AST.BaseExpr>(::ProblemExpression, frontend) {

/*
Magic numbers (https://docs.python.org/3/library/ast.html#ast.FormattedValue):
conversion is an integer:
-1: no formatting
115: !s string formatting
114: !r repr formatting
97: !a ascii formatting
*/
private val formattedValConversionNoFormatting = -1L
private val formattedValConversionString = 115L
private val formattedValConversionRepr = 114L
private val formattedValConversionASCII = 97L

override fun handleNode(node: Python.AST.BaseExpr): Expression {
return when (node) {
is Python.AST.Name -> handleName(node)
Expand Down Expand Up @@ -182,45 +168,116 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) :
return assignExpression
}

/**
* Translates a Python
* [`FormattedValue`](https://docs.python.org/3/library/ast.html#ast.FormattedValue) into an
* [Expression].
*
* We are handling the format handling, following [PEP 3101](https://peps.python.org/pep-3101).
*
* The following example
*
* ```python
* f"{value:.2f}"
* ```
*
* is modeled:
* 1. The value `value` is wrapped in a `format()` call.
* 2. The `format()` call has two arguments:
* - The value to format (`value`).
* - The format specification (`".2f"`).
*
* CPG Representation:
* - `CallExpression` node:
* - `callee`: `Reference` to `format`.
* - `arguments`:
* 1. A node representing `value`.
* 2. A node representing the string `".2f"`.
*/
private fun handleFormattedValue(node: Python.AST.FormattedValue): Expression {
if (node.format_spec != null) {
return newProblemExpression(
"Cannot handle formatted value with format_spec ${node.format_spec} yet",
rawNode = node
)
}
return when (node.conversion) {
formattedValConversionNoFormatting -> {
// No formatting, just return the value.
handle(node.value)
}
formattedValConversionString -> {
// String representation. wrap in str() call.
val strCall =
newCallExpression(newReference("str", rawNode = node), "str", rawNode = node)
strCall.addArgument(handle(node.value))
strCall
}
formattedValConversionRepr -> {
newProblemExpression(
"Cannot handle conversion '114: !r repr formatting', yet.",
rawNode = node
)
}
formattedValConversionASCII -> {
newProblemExpression(
"Cannot handle conversion '97: !a ascii formatting', yet.",
rawNode = node
)
/*
Magic numbers (https://docs.python.org/3/library/ast.html#ast.FormattedValue):
conversion is an integer:
-1: no formatting
115: !s string formatting
114: !r repr formatting
97: !a ascii formatting
*/
val formattedValConversionNoFormatting = -1L
val formattedValConversionString = 115L
val formattedValConversionRepr = 114L
val formattedValConversionASCII = 97L

val formatSpec = node.format_spec?.let { handle(it) }
val valueExpression = handle(node.value)
val conversion =
when (node.conversion) {
formattedValConversionNoFormatting -> {
// No formatting, just return the value.
valueExpression
}
formattedValConversionString -> {
// String representation: wrap in `str()` call.
val strCall =
newCallExpression(
callee = newReference(name = "str", rawNode = node),
fqn = "str",
rawNode = node
)
.implicit()
strCall.addArgument(valueExpression)
strCall
}
formattedValConversionRepr -> {
// Repr-String representation: wrap in `repr()` call.
val reprCall =
newCallExpression(
callee = newReference(name = "repr", rawNode = node),
fqn = "repr",
rawNode = node
)
.implicit()
reprCall.addArgument(valueExpression)
reprCall
}
formattedValConversionASCII -> {
// ASCII-String representation: wrap in `ascii()` call.
val asciiCall =
newCallExpression(
newReference("ascii", rawNode = node),
"ascii",
rawNode = node
)
.implicit()
asciiCall.addArgument(handle(node.value))
asciiCall
}
else ->
newProblemExpression(
problem =
"Cannot handle formatted value with conversion code ${node.conversion} yet",
rawNode = node
)
}
else ->
newProblemExpression(
"Cannot handle formatted value with conversion ${node.conversion} yet",
if (formatSpec != null) {
return newCallExpression(
callee = newReference(name = "format", rawNode = node),
fqn = "format",
rawNode = node
)
.implicit()
.apply {
addArgument(conversion)
addArgument(formatSpec)
}
}
return conversion
}

/**
* Translates a Python [`JoinedStr`](https://docs.python.org/3/library/ast.html#ast.JoinedStr)
* into a [Expression].
*/
private fun handleJoinedStr(node: Python.AST.JoinedStr): Expression {
val values = node.values.map(::handle)
return if (values.isEmpty()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import de.fraunhofer.aisec.cpg.graph.types.ObjectType
import de.fraunhofer.aisec.cpg.graph.types.SetType
import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker
import de.fraunhofer.aisec.cpg.passes.ControlDependenceGraphPass
import de.fraunhofer.aisec.cpg.query.value
import de.fraunhofer.aisec.cpg.sarif.Region
import de.fraunhofer.aisec.cpg.test.*
import java.nio.file.Path
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
/*
* Copyright (c) 2024, Fraunhofer AISEC. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* $$$$$$\ $$$$$$$\ $$$$$$\
* $$ __$$\ $$ __$$\ $$ __$$\
* $$ / \__|$$ | $$ |$$ / \__|
* $$ | $$$$$$$ |$$ |$$$$\
* $$ | $$ ____/ $$ |\_$$ |
* $$ | $$\ $$ | $$ | $$ |
* \$$$$$ |$$ | \$$$$$ |
* \______/ \__| \______/
*
*/
package de.fraunhofer.aisec.cpg.frontends.python.expressionHandler

import de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage
import de.fraunhofer.aisec.cpg.graph.*
import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration
import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression
import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator
import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression
import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal
import de.fraunhofer.aisec.cpg.test.analyzeAndGetFirstTU
import de.fraunhofer.aisec.cpg.test.assertLiteralValue
import de.fraunhofer.aisec.cpg.test.assertLocalName
import java.nio.file.Path
import kotlin.test.assertEquals
import kotlin.test.assertIs
import kotlin.test.assertNotNull
import org.junit.jupiter.api.BeforeAll
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.TestInstance

@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class FormattedValueHandlerTest {

private lateinit var topLevel: Path
private lateinit var result: TranslationUnitDeclaration

@BeforeAll
fun setup() {
topLevel = Path.of("src", "test", "resources", "python")
analyzeFile()
}

fun analyzeFile() {
result =
analyzeAndGetFirstTU(
listOf(topLevel.resolve("formatted_values.py").toFile()),
topLevel,
true
) {
it.registerLanguage<PythonLanguage>()
}
assertNotNull(result)
}

@Test
fun testFormattedValues() {
// Test for a = f'Number: {42:.2f}'
val aAssExpression = result.variables["a"]?.astParent
assertIs<AssignExpression>(aAssExpression)
val aExprRhs = aAssExpression.rhs.singleOrNull()
assertIs<BinaryOperator>(aExprRhs)
val aFormatCall = aExprRhs.rhs
assertIs<CallExpression>(aFormatCall)
assertLocalName("format", aFormatCall)
val aArguments = aFormatCall.arguments
assertEquals(2, aArguments.size)
assertIs<Literal<*>>(aArguments[0])
assertLiteralValue(42.toLong(), aArguments[0])
assertIs<Literal<*>>(aArguments[1])
assertLiteralValue(".2f", aArguments[1])

// Test for b = f'Hexadecimal: {255:#x}'
val bAssExpression = result.variables["b"]?.astParent
assertIs<AssignExpression>(bAssExpression)
val bExprRhs = bAssExpression.rhs.singleOrNull()
assertIs<BinaryOperator>(bExprRhs)
val bFormatCall = bExprRhs.rhs
assertIs<CallExpression>(bFormatCall)
assertLocalName("format", bFormatCall)
val bArguments = bFormatCall.arguments
assertEquals(2, bArguments.size)
assertIs<Literal<*>>(bArguments[0])
assertLiteralValue(255L.toLong(), bArguments[0])
// assertIs<Literal<*>>(bArguments[1])
assertLiteralValue("#x", bArguments[1])

// Test for c = f'String with conversion: {"Hello, world!"!r}'
val cAssExpression = result.variables["c"]?.astParent
assertIs<AssignExpression>(cAssExpression)
val cExprRhs = cAssExpression.rhs.singleOrNull()
assertIs<BinaryOperator>(cExprRhs)
val cConversionCall = cExprRhs.rhs
assertIs<CallExpression>(cConversionCall)
assertLocalName("repr", cConversionCall)
val cArguments = cConversionCall.arguments.singleOrNull()
assertNotNull(cArguments)
assertLiteralValue("Hello, world!", cArguments)

// Test for d = f'ASCII representation: {"50$"!a}'
val dAssExpression = result.variables["d"]?.astParent
assertIs<AssignExpression>(dAssExpression)
val dExprRhs = dAssExpression.rhs.singleOrNull()
assertIs<BinaryOperator>(dExprRhs)
val dConversionCall = dExprRhs.rhs
assertIs<CallExpression>(dConversionCall)
assertLocalName("ascii", dConversionCall)
val dArguments = dConversionCall.arguments.singleOrNull()
assertNotNull(dArguments)
assertLiteralValue("50$", dArguments)

// Test for e = f'Combined: {42!s:10}'
// This is translated to `'Combined: ' + format(str(b), "10")`
val eAssExpression = result.variables["e"]?.astParent
assertIs<AssignExpression>(eAssExpression)
val eExprRhs = eAssExpression.rhs.singleOrNull()
assertIs<BinaryOperator>(eExprRhs)
val eFormatCall = eExprRhs.rhs
assertIs<CallExpression>(eFormatCall)
assertLocalName("format", eFormatCall)
val eArguments = eFormatCall.arguments
assertEquals(2, eArguments.size)
val eConversionCall = eArguments[0]
assertIs<CallExpression>(eConversionCall)
assertLocalName("str", eConversionCall)
assertLiteralValue("42".toLong(), eConversionCall.arguments.singleOrNull())
assertLiteralValue("10", eArguments[1])
}
}
3 changes: 2 additions & 1 deletion cpg-language-python/src/test/resources/python/datatypes.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,5 @@
"e": "f"
}
e = f'Values of a: {a} and b: {b!s}'
f = a[1:3:2]
f = a[1:3:2]

Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
a = f'Number: {42:.2f}'
b = f'Hexadecimal: {255:#x}'
c = f'String with conversion: {"Hello, world!"!r}'
d = f'ASCII representation: {"50$"!a}'
e = f'Combined: {42!s:10}'
Original file line number Diff line number Diff line change
Expand Up @@ -493,6 +493,7 @@ class Application : Callable<Int> {
.optionalLanguage("de.fraunhofer.aisec.cpg.frontends.typescript.TypeScriptLanguage")
.optionalLanguage("de.fraunhofer.aisec.cpg.frontends.ruby.RubyLanguage")
.optionalLanguage("de.fraunhofer.aisec.cpg.frontends.jvm.JVMLanguage")
.optionalLanguage("de.fraunhofer.aisec.cpg.frontends.ini.IniFileLanguage")
.loadIncludes(loadIncludes)
.addIncludesToGraph(loadIncludes)
.debugParser(DEBUG_PARSER)
Expand Down

0 comments on commit 150ad6a

Please sign in to comment.