Next iteration of tests fixing

 On branch feature/fix-tests
 Changes to be committed:
	modified:   .github/workflows/ci.yml
	modified:   Makefile
	modified:   quinn/transformations.py

.github/workflows/ci.yml

@@ -74,7 +74,7 @@ jobs:
             poetry-${{ hashFiles('**/poetry.lock') }}
 
       - name: Install dependencies
-        run: make install_deps
+        run: make install_pure
         if: steps.cache.outputs.cache-hit != 'true'
 
       - name: Change PySpark to version ${{ matrix.pyspark-version }}

Makefile

@@ -1,5 +1,9 @@
 # COMMON CLI COMMANDS FOR DEVELOPMENT 
 
+.PHONY: install_pure
+install_pure:
+	@poetry install
+
 .PHONY: install_deps
 install_deps:
 	@poetry install --with=development,linting,testing,docs

quinn/transformations.py

@@ -3,7 +3,7 @@
 import re
 from collections.abc import Callable
 
-from pyspark.sql import DataFrame
+from pyspark.sql import DataFrame, SparkSession
 from pyspark.sql import functions as F  # noqa: N812
 from pyspark.sql.types import ArrayType, MapType, StructField, StructType
 
@@ -236,7 +236,12 @@ def fix_nullability(field: StructField, result_dict: dict) -> None:
     for field in output.schema:
         fix_nullability(field, result_dict)
 
-    return output.sparkSession.createDataFrame(output.rdd, output.schema)
+    spark = SparkSession.getActiveSession()
+
+    if spark is None:
+        spark = SparkSession.builder.getOrCreate()
+
+    return spark.sparkSession.createDataFrame(output.rdd, output.schema)
 
 
 def flatten_struct(df: DataFrame, col_name: str, separator: str = ":") -> DataFrame: