Major overhaul of the python language frontend (#513)

Co-authored-by: Banse, Christian <[email protected]>
Co-authored-by: Konrad Weiss <[email protected]>

.github/workflows/build.yml

@@ -62,19 +62,25 @@ jobs:
         id: determine_version
       - name: Install JEP
         run: |
-          pip3 install jep==4.0.0
+          pip3 install jep==$(grep "black.ninia:jep" cpg-library/build.gradle.kts | cut -d ":" -f 3 | cut -d "\"" -f 1)
           sudo cp /opt/hostedtoolcache/Python/3.9*/x64/lib/python3.9/site-packages/jep/libjep.so /usr/lib/
+      - name: Install pycodestyle
+        run: |
+            pip3 install pycodestyle
+      - name: Run pycodestyle
+        run: |
+            find cpg-library/src/main/python -iname "*.py" -exec pycodestyle \{\} \;
       - name: Build ${{ steps.determine_version.outputs.version }}
         run: |
           if [ "$SONAR_TOKEN" != "" ]
           then
-            ./gradlew --parallel -Pversion=$VERSION -Pexperimental -PexperimentalTypeScript -Pintegration build sonarqube \
+            ./gradlew --parallel -Pversion=$VERSION -Pexperimental -PexperimentalTypeScript -PexperimentalPython -Pintegration build sonarqube \
             -Dsonar.projectKey=Fraunhofer-AISEC_cpg \
             -Dsonar.organization=fraunhofer-aisec \
             -Dsonar.host.url=https://sonarcloud.io \
             -Dsonar.login=$SONAR_TOKEN
           else
-            ./gradlew --parallel -Pversion=$VERSION -Pexperimental -PexperimentalTypeScript -Pintegration build
+            ./gradlew --parallel -Pversion=$VERSION -Pexperimental -PexperimentalTypeScript -PexperimentalPython -Pintegration build
           fi
         id: build
         env:
@@ -91,7 +97,7 @@ jobs:
         if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, 'beta')      
         run: |
           export ORG_GRADLE_PROJECT_signingKey=`echo ${{ secrets.GPG_PRIVATE_KEY }} | base64 -d`
-          ./gradlew -Dorg.gradle.internal.publish.checksums.insecure=true --parallel -Pversion=$VERSION build signMavenPublication publish javadoc
+          ./gradlew -Dorg.gradle.internal.publish.checksums.insecure=true --parallel -Pversion=$VERSION -PexperimentalTypeScript -PexperimentalPython signMavenPublication build publish javadoc
         env:
           VERSION: ${{ steps.determine_version.outputs.version }}
           ORG_GRADLE_PROJECT_signingPassword: ${{ secrets.GPG_PASSWORD }}

README.md

@@ -48,17 +48,17 @@ The library can be used on the command line using the `cpg-console` subproject.
 
 ### Usage of Experimental Languages
 
-Some languages, such as Golang are marked as experimental and depend on other native libraries. These are NOT YET bundled in the release jars (with exception of TypeScript), so you need to build them manually using the property `-Pexperimental` when using tasks such as `build` or `test`. For typescript, please use `-PexperimentalTypeScript`.
+Some languages, such as Golang are marked as experimental and depend on other native libraries. These are NOT YET bundled in the release jars (with exception of TypeScript), so you need to build them manually using the property `-Pexperimental` when using tasks such as `build` or `test`. For typescript, please use `-PexperimentalTypeScript`. Use `-PexperimentalPython` for Python support, respectively.
 
 #### Golang
 
 In the case of Golang, the necessary native code can be found in the `src/main/golang` folder. Gradle should automatically find JNI headers and stores the finished library in the `src/main/golang` folder. This currently only works for Linux and macOS. In order to use it in an external project, the resulting library needs to be placed somewhere in `java.library.path`. 
 
 #### Python
 
-You need to install [jep](https://github.com/ninia/jep/). This can either be system wide or in a virtual environment. Furthermore, the python source, which are located in `src/main/python` need to be present in a directory with that name relative to where you execute or use CPG. We are working on extracting this into an actual python module, similar to jep. Currently, only Python 3.9 is supported.
+You need to install [jep](https://github.com/ninia/jep/). This can either be system wide or in a virtual environment. Your jep version hast to match the version used by CPG (see [build.gradle.kts](./cpg-library/build.gradle.kts)).
 
-Through the `JepSingleton`, the CPG library will look for well known paths on Linux and OS X. `JepSingleton` will prefer a virtualenv with the name `cpg`, this can be adjusted with the environment variable `CPG_PYTHON_VIRTUALENV`.
+Currently, only Python 3.9 is supported.
 
 ##### System Wide
 
@@ -70,6 +70,8 @@ Follow the instructions at https://github.com/ninia/jep/wiki/Getting-Started#ins
 - `source ~/.virtualenvs/cpg/bin/activate`
 - `pip3 install jep`
 
+Through the `JepSingleton`, the CPG library will look for well known paths on Linux and OS X. `JepSingleton` will prefer a virtualenv with the name `cpg`, this can be adjusted with the environment variable `CPG_PYTHON_VIRTUALENV`.
+
 #### TypeScript
 
 For parsing TypeScript, the necessary NodeJS-based code can be found in the `src/main/nodejs` directory of the `cpg-library` folder. Gradle should build the script automatically, provided NodeJS (>=16) is installed. The bundles script will be placed inside the jar's resources and should work out of the box.

cpg-library/build.gradle.kts

@@ -96,6 +96,10 @@ tasks.named<Test>("test") {
         if (!project.hasProperty("experimentalTypeScript")) {
             excludeTags("experimentalTypeScript")
         }
+
+        if (!project.hasProperty("experimentalPython")) {
+            excludeTags("experimentalPython")
+        }
     }
     maxHeapSize = "4048m"
 }
@@ -149,6 +153,16 @@ if (project.hasProperty("experimental")) {
     }
 }
 
+if (project.hasProperty("experimentalPython")) {
+    // add python source code to resources
+    tasks {
+        processResources {
+            from("src/main/python/")
+            include("CPGPython/*.py", "cpg.py")
+        }
+    }
+}
+
 if (project.hasProperty("experimentalTypeScript")) {
     tasks.processResources {
         dependsOn(yarnBuild)
@@ -193,8 +207,10 @@ dependencies {
     implementation("org.jetbrains.kotlin:kotlin-stdlib")
     implementation("org.jetbrains.kotlin:kotlin-reflect")
 
-    // jep for python support
-    api("black.ninia:jep:4.0.0")
+    if(project.hasProperty("experimentalPython")) {
+        // jep for python support
+        api("black.ninia:jep:4.0.0")
+    }
 
     // JUnit
     testImplementation("org.jetbrains.kotlin:kotlin-test")

cpg-library/src/main/java/de/fraunhofer/aisec/cpg/TranslationManager.kt

@@ -131,7 +131,8 @@ private constructor(
      *
      * @param result the translation result that is being mutated
      * @param config the translation configuration
-     * @throws TranslationException if the language front-end runs into an error and [TranslationConfiguration.failOnError]
+     * @throws TranslationException if the language front-end runs into an error and
+     * [TranslationConfiguration.failOnError]
      * * is `true`.
      */
     @Throws(TranslationException::class)

cpg-library/src/main/java/de/fraunhofer/aisec/cpg/frontends/python/JepSingleton.kt

@@ -26,20 +26,92 @@
 package de.fraunhofer.aisec.cpg.frontends.python
 
 import java.io.File
+import java.net.JarURLConnection
+import jep.JepConfig
 import jep.MainInterpreter
+import org.slf4j.LoggerFactory
 
 /**
  * Takes care of configuring Jep according to some well known paths on popular operating systems.
  */
 object JepSingleton {
+    var config = JepConfig()
+
+    private val LOGGER = LoggerFactory.getLogger(javaClass)
+
     init {
+        val tempFileHolder = PyTempFileHolder()
+        val classLoader = javaClass
+        val pyInitFile = classLoader.getResource("/CPGPython/__init__.py")
+
+        config.redirectStdErr(System.err)
+        config.redirectStdout(System.out)
+
+        if (pyInitFile?.protocol == "file") {
+            LOGGER.debug(
+                "Found the CPGPython module using a \"file\" resource. Using python code directly."
+            )
+            // we can point JEP to the folder and get better debug messages with python source code
+            // locations
+
+            // we want to have the parent folder of "CPGPython" so that we can do "import CPGPython"
+            // in python
+            var pyFolder = pyInitFile.file.dropLastWhile { it != File.separatorChar }
+            pyFolder = pyFolder.dropLast(1)
+            pyFolder = pyFolder.dropLastWhile { it != File.separatorChar }
+            config.addIncludePaths(pyFolder)
+        } else {
+            val targetFolder = tempFileHolder.pyFolder
+            config.addIncludePaths(tempFileHolder.pyFolder.toString())
+
+            // otherwise, we are probably running inside a JAR, so we try to extract our files
+            // out of the jar into a temporary folder
+            val jarURL = pyInitFile?.openConnection() as? JarURLConnection
+            val jar = jarURL?.jarFile
+
+            if (jar == null) {
+                LOGGER.error(
+                    "Could not extract CPGPython out of the jar. The python frontend will probably not work."
+                )
+            } else {
+                LOGGER.info(
+                    "Using JAR connection to {} to extract files into {}",
+                    jar.name,
+                    targetFolder
+                )
+
+                // we are only interested in the CPGPython directory
+                val entries = jar.entries().asSequence().filter { it.name.contains("CPGPython") }
+
+                entries.forEach { entry ->
+                    LOGGER.debug("Extracting entry: {}", entry.name)
+
+                    // resolve target files relatively to our target folder. They are already
+                    // prefixed with CPGPython/
+                    val targetFile = targetFolder.resolve(entry.name).toFile()
+
+                    // make sure to create directories along the way
+                    if (entry.isDirectory) {
+                        targetFile.mkdirs()
+                    } else {
+                        // copy the contents into the temp folder
+                        jar.getInputStream(entry).use { input ->
+                            targetFile.outputStream().use { output -> input.copyTo(output) }
+                        }
+                    }
+                }
+            }
+        }
+
         if (System.getenv("CPG_JEP_LIBRARY") != null) {
             val library = File(System.getenv("CPG_JEP_LIBRARY"))
             if (library.exists()) {
                 MainInterpreter.setJepLibraryPath(library.path)
+                config.addIncludePaths(
+                    library.toPath().parent.parent.toString()
+                ) // this assumes that the python code is also at the library's location
             }
         } else {
-
             var virtualEnv = "cpg"
 
             if (System.getenv("CPG_PYTHON_VIRTUALENV") != null) {
@@ -64,6 +136,9 @@ object JepSingleton {
                     // calls
                     // to setJepLibraryPath and co result in failures.
                     MainInterpreter.setJepLibraryPath(it.path)
+                    config.addIncludePaths(
+                        it.toPath().parent.parent.toString()
+                    ) // this assumes that the python code is also at the library's location
                 }
             }
         }

cpg-library/src/main/java/de/fraunhofer/aisec/cpg/frontends/python/PyTempFileHolder.kt

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2021, Fraunhofer AISEC. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *                    $$$$$$\  $$$$$$$\   $$$$$$\
+ *                   $$  __$$\ $$  __$$\ $$  __$$\
+ *                   $$ /  \__|$$ |  $$ |$$ /  \__|
+ *                   $$ |      $$$$$$$  |$$ |$$$$\
+ *                   $$ |      $$  ____/ $$ |\_$$ |
+ *                   $$ |  $$\ $$ |      $$ |  $$ |
+ *                   \$$$$$   |$$ |      \$$$$$   |
+ *                    \______/ \__|       \______/
+ *
+ */
+package de.fraunhofer.aisec.cpg.frontends.python
+
+import java.nio.file.Files
+import java.nio.file.Path
+
+class PyTempFileHolder {
+    // create temporary file and folder
+    var pyZipOnDisk: Path = Files.createTempFile("cpg_python", ".zip")
+    var pyFolder: Path = Files.createTempDirectory("cpg_python")
+
+    protected fun finalize() {
+        // clean up once no longer used
+        pyZipOnDisk.toFile().delete()
+        // pyFolder.toFile().deleteRecursively() // TODO
+    }
+}

cpg-library/src/main/java/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt

@@ -34,16 +34,17 @@ import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration
 import de.fraunhofer.aisec.cpg.passes.scopes.ScopeManager
 import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation
 import java.io.File
-import java.lang.Exception
 import java.nio.file.Path
-import jep.*
+import jep.JepException
+import jep.SubInterpreter
 
 @ExperimentalPython
 class PythonLanguageFrontend(config: TranslationConfiguration, scopeManager: ScopeManager?) :
     LanguageFrontend(config, scopeManager, ".") {
     companion object {
         @kotlin.jvm.JvmField var PY_EXTENSIONS: List<String> = listOf(".py")
     }
+    private val jep = JepSingleton // configure Jep
 
     @Throws(TranslationException::class)
     override fun parse(file: File): TranslationUnitDeclaration {
@@ -87,24 +88,20 @@ class PythonLanguageFrontend(config: TranslationConfiguration, scopeManager: Sco
             }
         }
 
-        if (!found) {
-            log.error("Could not find cpg.py")
-            throw TranslationException(
-                "Could not find cpg.py. We expect it to be either in the current working directory, in src/main/python/cpg.py or in cpg-library/src/main/python/cpg.py."
-            )
-        }
-
         val tu: TranslationUnitDeclaration
         var interp: SubInterpreter? = null
         try {
-            JepSingleton // configure Jep
-            interp =
-                SubInterpreter(JepConfig().redirectStdErr(System.err).redirectStdout(System.out))
-
-            // TODO: extract cpg.py in a real python module with multiple files
+            interp = SubInterpreter(jep.config)
 
             // load script
-            interp.runScript(entryScript.toString())
+            if (found) {
+                interp.runScript(entryScript.toString())
+            } else {
+                // fall back to the cpg.py in the class's resources
+                val classLoader = javaClass
+                val pyInitFile = classLoader.getResource("/cpg.py")
+                interp.exec(pyInitFile?.readText())
+            }
 
             // run python function parse_code()
             tu = interp.invoke("parse_code", code, path, this) as TranslationUnitDeclaration

cpg-library/src/main/java/de/fraunhofer/aisec/cpg/passes/CallResolver.java

@@ -1212,9 +1212,7 @@ private void resolveConstructExpression(ConstructExpression constructExpression)
       }
     }
 
-    if (recordDeclaration != null
-        && recordDeclaration.getCode() != null
-        && !recordDeclaration.getCode().isEmpty()) {
+    if (recordDeclaration != null) {
       ConstructorDeclaration constructor =
           getConstructorDeclaration(constructExpression, recordDeclaration);
       constructExpression.setConstructor(constructor);