From 33872299c2e6f2b363928a47353ffef665efe305 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Fri, 31 Jan 2025 11:09:05 +0100 Subject: [PATCH 01/11] Create scope for collection comprehensions --- .../kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt | 1 + .../de/fraunhofer/aisec/cpg/graph/TypeBuilder.kt | 2 +- .../cpg/frontends/python/ExpressionHandler.kt | 14 +++++++++++--- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt index ad4b11a93c..fcf96a9440 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt @@ -242,6 +242,7 @@ class ScopeManager : ScopeProvider { is TryStatement, is IfStatement, is CatchClause, + is CollectionComprehension, is Block -> LocalScope(nodeToScope) is FunctionDeclaration -> FunctionScope(nodeToScope) is RecordDeclaration -> RecordScope(nodeToScope) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/TypeBuilder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/TypeBuilder.kt index 3a4e343b7e..e8ff3b326c 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/TypeBuilder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/TypeBuilder.kt @@ -101,7 +101,7 @@ fun LanguageProvider.objectType( ): Type { // First, we check, whether this is a built-in type, to avoid necessary allocations of simple // types - val builtIn = language?.getSimpleTypeOf(name.toString()) + val builtIn = language.getSimpleTypeOf(name.toString()) if (builtIn != null) { return builtIn } diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt index 3da8338b34..48c4f8e7d1 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt @@ -106,9 +106,11 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : */ private fun handleGeneratorExp(node: Python.AST.GeneratorExp): CollectionComprehension { return newCollectionComprehension(rawNode = node).apply { + ctx?.scopeManager?.enterScope(this) statement = handle(node.elt) comprehensionExpressions += node.generators.map { handleComprehension(it, node) } type = objectType("Generator") + ctx?.scopeManager?.leaveScope(this) } } @@ -118,9 +120,11 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : */ private fun handleListComprehension(node: Python.AST.ListComp): CollectionComprehension { return newCollectionComprehension(rawNode = node).apply { + ctx?.scopeManager?.enterScope(this) statement = handle(node.elt) comprehensionExpressions += node.generators.map { handleComprehension(it, node) } - type = objectType("list") // TODO: Replace this once we have dedicated types + type = objectType("list") + ctx?.scopeManager?.leaveScope(this) } } @@ -130,9 +134,11 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : */ private fun handleSetComprehension(node: Python.AST.SetComp): CollectionComprehension { return newCollectionComprehension(rawNode = node).apply { + ctx?.scopeManager?.enterScope(this) this.statement = handle(node.elt) this.comprehensionExpressions += node.generators.map { handleComprehension(it, node) } - this.type = objectType("set") // TODO: Replace this once we have dedicated types + this.type = objectType("set") + ctx?.scopeManager?.leaveScope(this) } } @@ -142,6 +148,7 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : */ private fun handleDictComprehension(node: Python.AST.DictComp): CollectionComprehension { return newCollectionComprehension(rawNode = node).apply { + ctx?.scopeManager?.enterScope(this) this.statement = newKeyValueExpression( key = handle(node.key), @@ -149,7 +156,8 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : rawNode = node, ) this.comprehensionExpressions += node.generators.map { handleComprehension(it, node) } - this.type = objectType("dict") // TODO: Replace this once we have dedicated types + this.type = objectType("dict") + ctx?.scopeManager?.leaveScope(this) } } From 209c218f0b3fe88f97e50496875b1bf7883431c4 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Fri, 31 Jan 2025 11:31:23 +0100 Subject: [PATCH 02/11] generate local variables in python add declarations pass for comprehensions --- .../cpg/passes/PythonAddDeclarationsPass.kt | 37 +++++++++++++++++-- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt index d6760829c4..13dd938e07 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt @@ -37,6 +37,7 @@ import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.scopes.RecordScope import de.fraunhofer.aisec.cpg.graph.statements.ForEachStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.ComprehensionExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.InitializerListExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference @@ -67,15 +68,15 @@ class PythonAddDeclarationsPass(ctx: TranslationContext) : ComponentPass(ctx), L } /** - * This function checks for each [AssignExpression] whether there is already a matching variable - * or not. New variables can be one of: + * This function checks for each [AssignExpression], [ComprehensionExpression] and + * [ForEachStatement] whether there is already a matching variable or not. New variables can be + * one of: * - [FieldDeclaration] if we are currently in a record * - [VariableDeclaration] otherwise - * - * TODO: loops */ private fun handle(node: Node?) { when (node) { + is ComprehensionExpression -> handleComprehensionExpression(node) is AssignExpression -> handleAssignExpression(node) is ForEachStatement -> handleForEach(node) else -> { @@ -194,6 +195,34 @@ class PythonAddDeclarationsPass(ctx: TranslationContext) : ComponentPass(ctx), L this.base.name == scopeManager.currentMethod?.receiver?.name } + /** + * Generates a new [VariableDeclaration] for [Reference] (and those included in a + * [InitializerListExpression]) in the [ComprehensionExpression.variable]. + */ + private fun handleComprehensionExpression( + comprehensionExpression: ComprehensionExpression, + setAccessValue: Boolean = false, + ) { + when (val variable = comprehensionExpression.variable) { + is Reference -> { + if (setAccessValue) { + variable.access = AccessValues.WRITE + } + handleWriteToReference(variable) + } + is InitializerListExpression -> { + variable.initializers.forEach { + (it as? Reference)?.let { ref -> + if (setAccessValue) { + ref.access = AccessValues.WRITE + } + handleWriteToReference(ref) + } + } + } + } + } + /** * Generates a new [VariableDeclaration] if [target] is a [Reference] and there is no existing * declaration yet. From 1ff35b840e3fde59e0d4b38edb4e6293a31faf80 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Fri, 31 Jan 2025 11:31:44 +0100 Subject: [PATCH 03/11] Extend test to check fixes --- .../frontends/python/ExpressionHandlerTest.kt | 23 +++++++++++++++++-- .../test/resources/python/comprehension.py | 1 + 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt index 2f81c21425..40fc81dd9b 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt @@ -26,6 +26,7 @@ package de.fraunhofer.aisec.cpg.frontends.python import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block @@ -36,6 +37,8 @@ import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference import de.fraunhofer.aisec.cpg.test.analyze import de.fraunhofer.aisec.cpg.test.assertLiteralValue import de.fraunhofer.aisec.cpg.test.assertLocalName +import de.fraunhofer.aisec.cpg.test.assertNotRefersTo +import de.fraunhofer.aisec.cpg.test.assertRefersTo import java.nio.file.Path import kotlin.test.* @@ -57,14 +60,30 @@ class ExpressionHandlerTest { assertIs(singleWithIfAssignment) val singleWithIf = singleWithIfAssignment.rhs[0] assertIs(singleWithIf) - assertIs(singleWithIf.statement) + val fooCall = singleWithIf.statement + assertIs(fooCall) + val usageI = fooCall.arguments[0] + assertIs(usageI) assertEquals(1, singleWithIf.comprehensionExpressions.size) - assertLocalName("i", singleWithIf.comprehensionExpressions[0].variable) + val variableI = singleWithIf.comprehensionExpressions[0].variable + assertIs(variableI) + assertLocalName("i", variableI) + val declI = variableI.refersTo + assertIs(declI) + assertEquals(singleWithIf, declI.scope?.astNode) assertIs(singleWithIf.comprehensionExpressions[0].iterable) assertLocalName("x", singleWithIf.comprehensionExpressions[0].iterable) val ifPredicate = singleWithIf.comprehensionExpressions[0].predicate assertIs(ifPredicate) assertEquals("==", ifPredicate.operatorCode) + assertRefersTo(usageI, declI) + + val fooIOutside = body.statements[4] + assertIs(fooIOutside) + val outsideI = fooIOutside.arguments[0] + assertIs(outsideI) + assertLocalName("i", outsideI) + assertNotRefersTo(outsideI, declI) val singleWithoutIfAssignment = body.statements[1] assertIs(singleWithoutIfAssignment) diff --git a/cpg-language-python/src/test/resources/python/comprehension.py b/cpg-language-python/src/test/resources/python/comprehension.py index 7d6f2568ad..155b5f91c8 100644 --- a/cpg-language-python/src/test/resources/python/comprehension.py +++ b/cpg-language-python/src/test/resources/python/comprehension.py @@ -6,6 +6,7 @@ def listComp(x, y): b = [foo(i) for i in x] c = {foo(i) for i in x if i == 10 if i < 20} d = [foo(i) for z in y if z in x for i in z if i == 10 ] + foo(i) def setComp(x, y): a = {foo(i) for i in x if i == 10} From 501d597482dc98ff9109d2b307d5fad58c9a4350 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Fri, 31 Jan 2025 11:40:53 +0100 Subject: [PATCH 04/11] Remove useless flag --- .../aisec/cpg/passes/PythonAddDeclarationsPass.kt | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt index 13dd938e07..bb647f689d 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt @@ -199,23 +199,16 @@ class PythonAddDeclarationsPass(ctx: TranslationContext) : ComponentPass(ctx), L * Generates a new [VariableDeclaration] for [Reference] (and those included in a * [InitializerListExpression]) in the [ComprehensionExpression.variable]. */ - private fun handleComprehensionExpression( - comprehensionExpression: ComprehensionExpression, - setAccessValue: Boolean = false, - ) { + private fun handleComprehensionExpression(comprehensionExpression: ComprehensionExpression) { when (val variable = comprehensionExpression.variable) { is Reference -> { - if (setAccessValue) { - variable.access = AccessValues.WRITE - } + variable.access = AccessValues.WRITE handleWriteToReference(variable) } is InitializerListExpression -> { variable.initializers.forEach { (it as? Reference)?.let { ref -> - if (setAccessValue) { - ref.access = AccessValues.WRITE - } + ref.access = AccessValues.WRITE handleWriteToReference(ref) } } From cb0b6603368f69a41b716496b744794e1952f81a Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Fri, 31 Jan 2025 10:11:22 +0100 Subject: [PATCH 05/11] Add test case --- .../frontends/python/ExpressionHandlerTest.kt | 42 +++++++++++++++++++ .../test/resources/python/comprehension.py | 8 +++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt index 40fc81dd9b..fe808256a3 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt @@ -32,6 +32,7 @@ import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.CollectionComprehension +import de.fraunhofer.aisec.cpg.graph.statements.expressions.InitializerListExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.KeyValueExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference import de.fraunhofer.aisec.cpg.test.analyze @@ -43,6 +44,47 @@ import java.nio.file.Path import kotlin.test.* class ExpressionHandlerTest { + + @Test + fun testComprehensionExpressionTuple() { + val topLevel = Path.of("src", "test", "resources", "python") + val result = + analyze(listOf(topLevel.resolve("comprehension.py").toFile()), topLevel, true) { + it.registerLanguage() + } + assertNotNull(result) + + val tupleComp = result.functions["tupleComp"] + assertNotNull(tupleComp) + + val body = tupleComp.body + assertIs(body) + val tupleAsVariableAssignment = body.statements[0] + assertIs(tupleAsVariableAssignment) + val tupleAsVariable = tupleAsVariableAssignment.rhs[0] + assertIs(tupleAsVariable) + val barCall = tupleAsVariable.statement + assertIs(barCall) + assertLocalName("bar", barCall) + val argK = barCall.arguments[0] + assertIs(argK) + assertLocalName("k", argK) + val argV = barCall.arguments[1] + assertIs(argV) + assertLocalName("v", argV) + assertEquals(1, tupleAsVariable.comprehensionExpressions.size) + val initializerListExpression = tupleAsVariable.comprehensionExpressions[0].variable + assertIs(initializerListExpression) + val variableK = initializerListExpression.initializers[0] + assertIs(variableK) + assertLocalName("k", variableK) + val variableV = initializerListExpression.initializers[1] + assertIs(variableV) + assertLocalName("v", variableV) + assertEquals(setOf(argK), variableK.nextDFG.toSet()) + assertEquals(setOf(argV), variableV.nextDFG.toSet()) + } + @Test fun testListComprehensions() { val topLevel = Path.of("src", "test", "resources", "python") diff --git a/cpg-language-python/src/test/resources/python/comprehension.py b/cpg-language-python/src/test/resources/python/comprehension.py index 155b5f91c8..3f1fbd400b 100644 --- a/cpg-language-python/src/test/resources/python/comprehension.py +++ b/cpg-language-python/src/test/resources/python/comprehension.py @@ -22,4 +22,10 @@ def dictComp(x, y): def generator(x, y): a = (i**2 for i in range(10) if i == 10) - b = (i**2 for i in range(10)) \ No newline at end of file + b = (i**2 for i in range(10)) + +def bar(k, v): + return k+v + +def tupleComp(x): + a = [bar(k, v) for (k, v) in x] \ No newline at end of file From 8b8fccdf872c441d47da15292651082dc0fd9db1 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Fri, 31 Jan 2025 11:47:16 +0100 Subject: [PATCH 06/11] Adapt test --- .../cpg/frontends/python/ExpressionHandlerTest.kt | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt index fe808256a3..62c47171d6 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt @@ -27,6 +27,7 @@ package de.fraunhofer.aisec.cpg.frontends.python import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration +import de.fraunhofer.aisec.cpg.graph.scopes.LocalScope import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block @@ -81,8 +82,18 @@ class ExpressionHandlerTest { val variableV = initializerListExpression.initializers[1] assertIs(variableV) assertLocalName("v", variableV) - assertEquals(setOf(argK), variableK.nextDFG.toSet()) - assertEquals(setOf(argV), variableV.nextDFG.toSet()) + + // Check that the declarations exist for the variables k and v + val declK = variableK.refersTo + assertIs(declK) + assertIs(declK.scope) + assertEquals(tupleAsVariable, declK.scope?.astNode) + assertRefersTo(argK, declK) + val declV = variableV.refersTo + assertIs(declV) + assertIs(declV.scope) + assertEquals(tupleAsVariable, declV.scope?.astNode) + assertRefersTo(argV, declV) } @Test From c7893a622fcc2ba133a9cc3365682790e999d10d Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Fri, 31 Jan 2025 11:58:14 +0100 Subject: [PATCH 07/11] Test resolution of param --- .../cpg/frontends/python/ExpressionHandlerTest.kt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt index 62c47171d6..1e2ca0e28b 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt @@ -26,6 +26,7 @@ package de.fraunhofer.aisec.cpg.frontends.python import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.declarations.ParameterDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.scopes.LocalScope import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression @@ -106,6 +107,9 @@ class ExpressionHandlerTest { assertNotNull(result) val listComp = result.functions["listComp"] assertNotNull(listComp) + val paramX = listComp.parameters[0] + assertIs(paramX) + assertLocalName("x", paramX) val body = listComp.body assertIs(body) @@ -124,8 +128,10 @@ class ExpressionHandlerTest { val declI = variableI.refersTo assertIs(declI) assertEquals(singleWithIf, declI.scope?.astNode) - assertIs(singleWithIf.comprehensionExpressions[0].iterable) - assertLocalName("x", singleWithIf.comprehensionExpressions[0].iterable) + val iterableX = singleWithIf.comprehensionExpressions[0].iterable + assertIs(iterableX) + assertLocalName("x", iterableX) + assertRefersTo(iterableX, paramX) val ifPredicate = singleWithIf.comprehensionExpressions[0].predicate assertIs(ifPredicate) assertEquals("==", ifPredicate.operatorCode) From 110c288bf0abf76c5d3f4521a142d5d6acf059ea Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Fri, 31 Jan 2025 12:03:26 +0100 Subject: [PATCH 08/11] Reduce c&p code --- .../cpg/frontends/python/ExpressionHandler.kt | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt index 48c4f8e7d1..1c25f2158e 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt @@ -99,18 +99,24 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : } } + private inline fun T.applyWithScope(block: T.() -> Unit): T { + return this.apply { + ctx?.scopeManager?.enterScope(this) + block() + ctx?.scopeManager?.leaveScope(this) + } + } + /** * Translates a Python * [`GeneratorExp`](https://docs.python.org/3/library/ast.html#ast.GeneratorExp) into a * [CollectionComprehension]. */ private fun handleGeneratorExp(node: Python.AST.GeneratorExp): CollectionComprehension { - return newCollectionComprehension(rawNode = node).apply { - ctx?.scopeManager?.enterScope(this) + return newCollectionComprehension(rawNode = node).applyWithScope { statement = handle(node.elt) comprehensionExpressions += node.generators.map { handleComprehension(it, node) } type = objectType("Generator") - ctx?.scopeManager?.leaveScope(this) } } @@ -119,12 +125,10 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : * into a [CollectionComprehension]. */ private fun handleListComprehension(node: Python.AST.ListComp): CollectionComprehension { - return newCollectionComprehension(rawNode = node).apply { - ctx?.scopeManager?.enterScope(this) + return newCollectionComprehension(rawNode = node).applyWithScope { statement = handle(node.elt) comprehensionExpressions += node.generators.map { handleComprehension(it, node) } type = objectType("list") - ctx?.scopeManager?.leaveScope(this) } } @@ -133,12 +137,10 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : * a [CollectionComprehension]. */ private fun handleSetComprehension(node: Python.AST.SetComp): CollectionComprehension { - return newCollectionComprehension(rawNode = node).apply { - ctx?.scopeManager?.enterScope(this) + return newCollectionComprehension(rawNode = node).applyWithScope { this.statement = handle(node.elt) this.comprehensionExpressions += node.generators.map { handleComprehension(it, node) } this.type = objectType("set") - ctx?.scopeManager?.leaveScope(this) } } @@ -147,8 +149,7 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : * into a [CollectionComprehension]. */ private fun handleDictComprehension(node: Python.AST.DictComp): CollectionComprehension { - return newCollectionComprehension(rawNode = node).apply { - ctx?.scopeManager?.enterScope(this) + return newCollectionComprehension(rawNode = node).applyWithScope { this.statement = newKeyValueExpression( key = handle(node.key), @@ -157,7 +158,6 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : ) this.comprehensionExpressions += node.generators.map { handleComprehension(it, node) } this.type = objectType("dict") - ctx?.scopeManager?.leaveScope(this) } } From 8011422ee653b62e79dd43f490ef7e6694ca2693 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Fri, 31 Jan 2025 12:05:35 +0100 Subject: [PATCH 09/11] add documentation --- .../aisec/cpg/frontends/python/ExpressionHandler.kt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt index 1c25f2158e..a3f1f50b92 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt @@ -99,6 +99,10 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : } } + /** + * Works similar to [apply] but before executing [block], it enters the scope for this object + * and afterward leaves the scope again. + */ private inline fun T.applyWithScope(block: T.() -> Unit): T { return this.apply { ctx?.scopeManager?.enterScope(this) From 9489ecf5280d497389b57f887f91ca086e4474b9 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Fri, 31 Jan 2025 13:28:13 +0100 Subject: [PATCH 10/11] Move applyWithScope to Node --- .../kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt | 12 ++++++++++++ .../aisec/cpg/frontends/python/ExpressionHandler.kt | 12 ------------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt index 7ada182428..d4fb769e55 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt @@ -369,3 +369,15 @@ abstract class Node : const val EMPTY_NAME = "" } } + +/** + * Works similar to [apply] but before executing [block], it enters the scope for this object and + * afterward leaves the scope again. + */ +inline fun T.applyWithScope(block: T.() -> Unit): T { + return this.apply { + ctx?.scopeManager?.enterScope(this) + block() + ctx?.scopeManager?.leaveScope(this) + } +} diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt index a3f1f50b92..6e41acdd1c 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt @@ -99,18 +99,6 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : } } - /** - * Works similar to [apply] but before executing [block], it enters the scope for this object - * and afterward leaves the scope again. - */ - private inline fun T.applyWithScope(block: T.() -> Unit): T { - return this.apply { - ctx?.scopeManager?.enterScope(this) - block() - ctx?.scopeManager?.leaveScope(this) - } - } - /** * Translates a Python * [`GeneratorExp`](https://docs.python.org/3/library/ast.html#ast.GeneratorExp) into a From 091041a8a1bf6c4b6c4c2cd3fd1abf99214c3907 Mon Sep 17 00:00:00 2001 From: Alexander Kuechler Date: Fri, 31 Jan 2025 13:31:47 +0100 Subject: [PATCH 11/11] python style --- .../aisec/cpg/frontends/python/ExpressionHandlerTest.kt | 8 ++++---- .../src/test/resources/python/comprehension.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt index 1e2ca0e28b..af73e5f2a9 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt @@ -56,7 +56,7 @@ class ExpressionHandlerTest { } assertNotNull(result) - val tupleComp = result.functions["tupleComp"] + val tupleComp = result.functions["tuple_comp"] assertNotNull(tupleComp) val body = tupleComp.body @@ -105,7 +105,7 @@ class ExpressionHandlerTest { it.registerLanguage() } assertNotNull(result) - val listComp = result.functions["listComp"] + val listComp = result.functions["list_comp"] assertNotNull(listComp) val paramX = listComp.parameters[0] assertIs(paramX) @@ -185,7 +185,7 @@ class ExpressionHandlerTest { it.registerLanguage() } assertNotNull(result) - val listComp = result.functions["setComp"] + val listComp = result.functions["set_comp"] assertNotNull(listComp) val body = listComp.body as? Block @@ -243,7 +243,7 @@ class ExpressionHandlerTest { it.registerLanguage() } assertNotNull(result) - val listComp = result.functions["dictComp"] + val listComp = result.functions["dict_comp"] assertNotNull(listComp) val body = listComp.body as? Block diff --git a/cpg-language-python/src/test/resources/python/comprehension.py b/cpg-language-python/src/test/resources/python/comprehension.py index 3f1fbd400b..c0bf329800 100644 --- a/cpg-language-python/src/test/resources/python/comprehension.py +++ b/cpg-language-python/src/test/resources/python/comprehension.py @@ -1,20 +1,20 @@ def foo(arg): return 7 -def listComp(x, y): +def list_comp(x, y): a = [foo(i) for i in x if i == 10] b = [foo(i) for i in x] c = {foo(i) for i in x if i == 10 if i < 20} d = [foo(i) for z in y if z in x for i in z if i == 10 ] foo(i) -def setComp(x, y): +def set_comp(x, y): a = {foo(i) for i in x if i == 10} b = {foo(i) for i in x} c = {foo(i) for i in x if i == 10 if i < 20} d = {foo(i) for z in y if z in x for i in z if i == 10 } -def dictComp(x, y): +def dict_comp(x, y): a = {i: foo(i) for i in x if i == 10} b = {i: foo(i) for i in x} c = {i: foo(i) for i in x if i == 10 if i < 20} @@ -27,5 +27,5 @@ def generator(x, y): def bar(k, v): return k+v -def tupleComp(x): +def tuple_comp(x): a = [bar(k, v) for (k, v) in x] \ No newline at end of file