From f33e319d2e886f8dfa77c54da7eeb21315b5cb3e Mon Sep 17 00:00:00 2001 From: SemyonSinchenko Date: Sat, 18 Nov 2023 21:26:51 +0000 Subject: [PATCH 1/9] Fix Lint badge Changes to be committed: modified: README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 62d96b8d..d1b1d537 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Quinn ![![image](https://github.com/MrPowers/quinn/workflows/build/badge.svg)](https://github.com/MrPowers/quinn/actions/workflows/ci.yml/badge.svg) -![![image](https://github.com/MrPowers/quinn/workflows/build/badge.svg)](https://github.com/MrPowers/quinn/actions/workflows/lint.yml/badge.svg) +![![image](https://github.com/MrPowers/quinn/workflows/build/badge.svg)](https://github.com/MrPowers/quinn/actions/workflows/lint.yaml/badge.svg) ![PyPI - Downloads](https://img.shields.io/pypi/dm/quinn) [![PyPI version](https://badge.fury.io/py/quinn.svg)](https://badge.fury.io/py/quinn) From 09c3fd4d5510d74f7d6bcd24568eb109cde712df Mon Sep 17 00:00:00 2001 From: SemyonSinchenko Date: Sat, 18 Nov 2023 21:35:34 +0000 Subject: [PATCH 2/9] Fix some tests On branch feature/fix-tests Changes to be committed: modified: pyproject.toml modified: quinn/schema_helpers.py modified: quinn/transformations.py modified: tests/test_split_columns.py --- pyproject.toml | 1 + quinn/schema_helpers.py | 4 ++-- quinn/transformations.py | 2 +- tests/test_split_columns.py | 4 +--- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ff72e82c..076f7aca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,6 +77,7 @@ ignore = [ "TCH003", # I have no idea what is it about "PLC1901", # Strange thing "UP007", # Not supported in py3.6 + "UP038", # Not supported in all py versions ] extend-exclude = ["tests", "docs"] diff --git a/quinn/schema_helpers.py b/quinn/schema_helpers.py index 9542a377..94032f02 100644 --- a/quinn/schema_helpers.py +++ b/quinn/schema_helpers.py @@ -54,7 +54,7 @@ def print_schema_as_code(dtype: T.DataType) -> str: def _repr_column(column: T.StructField) -> str: res = [] - if isinstance(column.dataType, (T.ArrayType | T.MapType | T.StructType)): + if isinstance(column.dataType, (T.ArrayType, T.MapType, T.StructType)): res.append(f'StructField(\n\t"{column.name}",') for line in print_schema_as_code(column.dataType).split("\n"): res.append("\n\t") @@ -164,6 +164,6 @@ def complex_fields(schema: T.StructType) -> dict[str, object]: return { field.name: field.dataType for field in schema.fields - if isinstance(field.dataType, (T.ArrayType | T.StructType | T.MapType)) + if isinstance(field.dataType, (T.ArrayType, T.StructType, T.MapType)) } diff --git a/quinn/transformations.py b/quinn/transformations.py index fbda2eee..0f79f2c7 100644 --- a/quinn/transformations.py +++ b/quinn/transformations.py @@ -220,7 +220,7 @@ def fix_nullability(field: StructField, result_dict: dict) -> None: return top_level_sorted_df is_nested: bool = any( - isinstance(i.dataType, StructType | ArrayType) + isinstance(i.dataType, (StructType, ArrayType)) for i in top_level_sorted_df.schema ) diff --git a/tests/test_split_columns.py b/tests/test_split_columns.py index e976df1d..0ddf8dc2 100644 --- a/tests/test_split_columns.py +++ b/tests/test_split_columns.py @@ -3,8 +3,6 @@ import chispa import pytest -from pyspark.errors.exceptions.captured import PythonException - @auto_inject_fixtures("spark") def test_split_columns(spark): @@ -52,5 +50,5 @@ def test_split_columns_strict(spark): delimiter="XX", new_col_names=["student_first_name", "student_middle_name", "student_last_name"], mode="strict", default="hi") - with pytest.raises(PythonException): + with pytest.raises(Exception): # there is no way to make it work for all the versions df2.show() From 48372f7d954687abad13d70ae8a68fd67fbe062c Mon Sep 17 00:00:00 2001 From: SemyonSinchenko Date: Sat, 18 Nov 2023 21:46:29 +0000 Subject: [PATCH 3/9] Next iteration of tests fixing On branch feature/fix-tests Changes to be committed: modified: .github/workflows/ci.yml modified: Makefile modified: quinn/transformations.py --- .github/workflows/ci.yml | 2 +- Makefile | 4 ++++ quinn/transformations.py | 9 +++++++-- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 24519179..beb251ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,7 +74,7 @@ jobs: poetry-${{ hashFiles('**/poetry.lock') }} - name: Install dependencies - run: make install_deps + run: make install_pure if: steps.cache.outputs.cache-hit != 'true' - name: Change PySpark to version ${{ matrix.pyspark-version }} diff --git a/Makefile b/Makefile index a13375ea..69b4d925 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,9 @@ # COMMON CLI COMMANDS FOR DEVELOPMENT +.PHONY: install_pure +install_pure: + @poetry install + .PHONY: install_deps install_deps: @poetry install --with=development,linting,testing,docs diff --git a/quinn/transformations.py b/quinn/transformations.py index 0f79f2c7..c0214e51 100644 --- a/quinn/transformations.py +++ b/quinn/transformations.py @@ -3,7 +3,7 @@ import re from collections.abc import Callable -from pyspark.sql import DataFrame +from pyspark.sql import DataFrame, SparkSession from pyspark.sql import functions as F # noqa: N812 from pyspark.sql.types import ArrayType, MapType, StructField, StructType @@ -236,7 +236,12 @@ def fix_nullability(field: StructField, result_dict: dict) -> None: for field in output.schema: fix_nullability(field, result_dict) - return output.sparkSession.createDataFrame(output.rdd, output.schema) + spark = SparkSession.getActiveSession() + + if spark is None: + spark = SparkSession.builder.getOrCreate() + + return spark.sparkSession.createDataFrame(output.rdd, output.schema) def flatten_struct(df: DataFrame, col_name: str, separator: str = ":") -> DataFrame: From 61aa25da8ffaefd8055bb4502d1afec84b184c0b Mon Sep 17 00:00:00 2001 From: SemyonSinchenko Date: Sat, 18 Nov 2023 21:49:13 +0000 Subject: [PATCH 4/9] Fix CI On branch feature/fix-tests Changes to be committed: modified: .github/workflows/ci.yml modified: Makefile --- .github/workflows/ci.yml | 2 +- Makefile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index beb251ea..3637d14e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,7 +74,7 @@ jobs: poetry-${{ hashFiles('**/poetry.lock') }} - name: Install dependencies - run: make install_pure + run: make install_pytest if: steps.cache.outputs.cache-hit != 'true' - name: Change PySpark to version ${{ matrix.pyspark-version }} diff --git a/Makefile b/Makefile index 69b4d925..5e631c85 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ # COMMON CLI COMMANDS FOR DEVELOPMENT -.PHONY: install_pure +.PHONY: install_pytest install_pure: - @poetry install + @poetry install --with=testing .PHONY: install_deps install_deps: From ad11b86f2076193cf3a9f6aa05d32b837789e04d Mon Sep 17 00:00:00 2001 From: SemyonSinchenko Date: Sat, 18 Nov 2023 21:53:19 +0000 Subject: [PATCH 5/9] Fix CI v2 On branch feature/fix-tests Changes to be committed: modified: .github/workflows/ci.yml modified: Makefile --- .github/workflows/ci.yml | 4 ++-- Makefile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3637d14e..7bce0af0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,8 +74,8 @@ jobs: poetry-${{ hashFiles('**/poetry.lock') }} - name: Install dependencies - run: make install_pytest - if: steps.cache.outputs.cache-hit != 'true' + run: make install_test + # if: steps.cache.outputs.cache-hit != 'true' - name: Change PySpark to version ${{ matrix.pyspark-version }} env: diff --git a/Makefile b/Makefile index 5e631c85..8938aed8 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ # COMMON CLI COMMANDS FOR DEVELOPMENT -.PHONY: install_pytest +.PHONY: install_test install_pure: - @poetry install --with=testing + @poetry install --with=development,testing .PHONY: install_deps install_deps: From a5473439aff80d5b26e0199adf250f645d12bde7 Mon Sep 17 00:00:00 2001 From: SemyonSinchenko Date: Sat, 18 Nov 2023 21:55:21 +0000 Subject: [PATCH 6/9] Fix CI final On branch feature/fix-tests Changes to be committed: modified: .github/workflows/ci.yml modified: Makefile --- .github/workflows/ci.yml | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7bce0af0..54ba31b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,7 +75,7 @@ jobs: - name: Install dependencies run: make install_test - # if: steps.cache.outputs.cache-hit != 'true' + if: steps.cache.outputs.cache-hit != 'true' - name: Change PySpark to version ${{ matrix.pyspark-version }} env: diff --git a/Makefile b/Makefile index 8938aed8..d3b1c42d 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ install_pure: @poetry install --with=development,testing .PHONY: install_deps -install_deps: +install_test: @poetry install --with=development,linting,testing,docs .PHONY: update_deps From a2dd5911cd15281fee4d5df6241e70f78623ce1e Mon Sep 17 00:00:00 2001 From: SemyonSinchenko Date: Sat, 18 Nov 2023 22:14:16 +0000 Subject: [PATCH 7/9] Relax restrictions to allow low py versions On branch feature/fix-tests Changes to be committed: modified: poetry.lock modified: pyproject.toml modified: quinn/transformations.py --- poetry.lock | 72 +++++++++++++++++++++++++++++++--------- pyproject.toml | 4 +-- quinn/transformations.py | 2 +- 3 files changed, 59 insertions(+), 19 deletions(-) diff --git a/poetry.lock b/poetry.lock index d01e5514..b8092273 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,9 +1,10 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. [[package]] name = "cached-property" version = "1.5.2" description = "A decorator for caching properties in classes." +category = "dev" optional = false python-versions = "*" files = [ @@ -15,6 +16,7 @@ files = [ name = "chispa" version = "0.9.4" description = "Pyspark test helper library" +category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -26,6 +28,7 @@ files = [ name = "click" version = "8.1.7" description = "Composable command line interface toolkit" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -41,6 +44,7 @@ importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." +category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -52,6 +56,7 @@ files = [ name = "exceptiongroup" version = "1.1.3" description = "Backport of PEP 654 (exception groups)" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -66,6 +71,7 @@ test = ["pytest (>=6)"] name = "ghp-import" version = "2.1.0" description = "Copy your docs directly to the gh-pages branch." +category = "dev" optional = false python-versions = "*" files = [ @@ -83,6 +89,7 @@ dev = ["flake8", "markdown", "twine", "wheel"] name = "griffe" version = "0.30.1" description = "Signatures for entire Python programs. Extract the structure, the frame, the skeleton of your project, to generate API documentation or find breaking changes in your API." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -98,6 +105,7 @@ colorama = ">=0.4" name = "importlib-metadata" version = "6.7.0" description = "Read metadata from Python packages" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -118,6 +126,7 @@ testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -129,6 +138,7 @@ files = [ name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -146,6 +156,7 @@ i18n = ["Babel (>=2.7)"] name = "markdown" version = "3.4.4" description = "Python implementation of John Gruber's Markdown." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -164,6 +175,7 @@ testing = ["coverage", "pyyaml"] name = "markdown-include" version = "0.8.1" description = "A Python-Markdown extension which provides an 'include' function" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -181,6 +193,7 @@ tests = ["pytest"] name = "markupsafe" version = "2.1.3" description = "Safely add untrusted strings to HTML/XML markup." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -250,6 +263,7 @@ files = [ name = "mergedeep" version = "1.3.4" description = "A deep merge function for 🐍." +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -261,6 +275,7 @@ files = [ name = "mkdocs" version = "1.5.3" description = "Project documentation with Markdown." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -293,6 +308,7 @@ min-versions = ["babel (==2.9.0)", "click (==7.0)", "colorama (==0.4)", "ghp-imp name = "mkdocs-autorefs" version = "0.4.1" description = "Automatically link across pages in MkDocs." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -308,6 +324,7 @@ mkdocs = ">=1.1" name = "mkdocs-gen-files" version = "0.4.0" description = "MkDocs plugin to programmatically generate documentation pages during the build" +category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -322,6 +339,7 @@ mkdocs = ">=1.0.3,<2.0.0" name = "mkdocs-literate-nav" version = "0.6.1" description = "MkDocs plugin to specify the navigation in Markdown instead of YAML" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -336,6 +354,7 @@ mkdocs = ">=1.0.3" name = "mkdocs-section-index" version = "0.3.8" description = "MkDocs plugin to allow clickable sections that lead to an index page" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -350,6 +369,7 @@ mkdocs = ">=1.2" name = "mkdocstrings" version = "0.22.0" description = "Automatic documentation from sources, for MkDocs." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -376,6 +396,7 @@ python-legacy = ["mkdocstrings-python-legacy (>=0.2.1)"] name = "mkdocstrings-python" version = "0.8.3" description = "A Python handler for mkdocstrings." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -391,6 +412,7 @@ mkdocstrings = ">=0.19" name = "packaging" version = "23.2" description = "Core utilities for Python packages" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -402,6 +424,7 @@ files = [ name = "pathspec" version = "0.11.2" description = "Utility library for gitignore style pattern matching of file paths." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -411,13 +434,14 @@ files = [ [[package]] name = "platformdirs" -version = "3.11.0" +version = "4.0.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "platformdirs-3.11.0-py3-none-any.whl", hash = "sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e"}, - {file = "platformdirs-3.11.0.tar.gz", hash = "sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3"}, + {file = "platformdirs-4.0.0-py3-none-any.whl", hash = "sha256:118c954d7e949b35437270383a3f2531e99dd93cf7ce4dc8340d3356d30f173b"}, + {file = "platformdirs-4.0.0.tar.gz", hash = "sha256:cb633b2bcf10c51af60beb0ab06d2f1d69064b43abf4c185ca6b28865f3f9731"}, ] [package.dependencies] @@ -431,6 +455,7 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-co name = "pluggy" version = "1.2.0" description = "plugin and hook calling mechanisms for python" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -449,6 +474,7 @@ testing = ["pytest", "pytest-benchmark"] name = "py4j" version = "0.10.9.7" description = "Enables Python programs to dynamically access arbitrary Java objects" +category = "dev" optional = false python-versions = "*" files = [ @@ -460,6 +486,7 @@ files = [ name = "pymdown-extensions" version = "10.2.1" description = "Extension pack for Python Markdown." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -478,6 +505,7 @@ extra = ["pygments (>=2.12)"] name = "pyspark" version = "3.4.1" description = "Apache Spark Python API" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -496,13 +524,14 @@ sql = ["numpy (>=1.15)", "pandas (>=1.0.5)", "pyarrow (>=1.0.0)"] [[package]] name = "pytest" -version = "7.4.2" +version = "7.4.3" description = "pytest: simple powerful testing with Python" +category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.4.2-py3-none-any.whl", hash = "sha256:1d881c6124e08ff0a1bb75ba3ec0bfd8b5354a01c194ddd5a0a870a48d99b002"}, - {file = "pytest-7.4.2.tar.gz", hash = "sha256:a766259cfab564a2ad52cb1aae1b881a75c3eb7e34ca3779697c23ed47c47069"}, + {file = "pytest-7.4.3-py3-none-any.whl", hash = "sha256:0d009c083ea859a71b76adf7c1d502e4bc170b80a8ef002da5806527b9591fac"}, + {file = "pytest-7.4.3.tar.gz", hash = "sha256:d989d136982de4e3b29dabcc838ad581c64e8ed52c11fbe86ddebd9da0818cd5"}, ] [package.dependencies] @@ -521,6 +550,7 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no name = "pytest-describe" version = "2.1.0" description = "Describe-style plugin for pytest" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -535,6 +565,7 @@ pytest = ">=4.6,<8" name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" +category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -549,6 +580,7 @@ six = ">=1.5" name = "pyyaml" version = "6.0.1" description = "YAML parser and emitter for Python" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -608,6 +640,7 @@ files = [ name = "pyyaml-env-tag" version = "0.1" description = "A custom YAML tag for referencing environment variables in YAML files. " +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -622,6 +655,7 @@ pyyaml = "*" name = "ruff" version = "0.0.291" description = "An extremely fast Python linter, written in Rust." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -646,19 +680,21 @@ files = [ [[package]] name = "semver" -version = "3.0.1" +version = "3.0.2" description = "Python helper for Semantic Versioning (https://semver.org)" +category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "semver-3.0.1-py3-none-any.whl", hash = "sha256:2a23844ba1647362c7490fe3995a86e097bb590d16f0f32dfc383008f19e4cdf"}, - {file = "semver-3.0.1.tar.gz", hash = "sha256:9ec78c5447883c67b97f98c3b6212796708191d22e4ad30f4570f840171cbce1"}, + {file = "semver-3.0.2-py3-none-any.whl", hash = "sha256:b1ea4686fe70b981f85359eda33199d60c53964284e0cfb4977d243e37cf4bf4"}, + {file = "semver-3.0.2.tar.gz", hash = "sha256:6253adb39c70f6e51afed2fa7152bcd414c411286088fb4b9effb133885ab4cc"}, ] [[package]] name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -670,6 +706,7 @@ files = [ name = "tomli" version = "2.0.1" description = "A lil' TOML parser" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -679,19 +716,21 @@ files = [ [[package]] name = "typing-extensions" -version = "4.8.0" -description = "Backported and Experimental Type Hints for Python 3.8+" +version = "4.7.1" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "dev" optional = false -python-versions = ">=3.8" +python-versions = ">=3.7" files = [ - {file = "typing_extensions-4.8.0-py3-none-any.whl", hash = "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0"}, - {file = "typing_extensions-4.8.0.tar.gz", hash = "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef"}, + {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, + {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, ] [[package]] name = "watchdog" version = "3.0.0" description = "Filesystem events monitoring" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -731,6 +770,7 @@ watchmedo = ["PyYAML (>=3.10)"] name = "zipp" version = "3.15.0" description = "Backport of pathlib-compatible object wrapper for zip files" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -745,4 +785,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = ">=3.7,<4.0" -content-hash = "5559a0b7175528b497b00d5e68b5841721e49230e371ad307c74c64e82a55d9a" +content-hash = "41791e6258914ac7f51b4ce6759ae8a03c8345313ea50c0c650293797e13c571" diff --git a/pyproject.toml b/pyproject.toml index 076f7aca..58b03671 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ pytest-describe = "^2" [tool.poetry.group.linting.dependencies] ruff = "^0.0.291" -typing-extensions = {version = "^4.8.0", python = ">=3.8,<4.0"} +typing-extensions = {version = "^4", python = ">=3.6,<4.0"} [tool.poetry.group.docs.dependencies] mkdocstrings-python = "^0.8.3" @@ -55,7 +55,7 @@ mkdocs-gen-files = "^0.4.0" mkdocs-literate-nav = "^0.6.0" mkdocs-section-index = "^0.3.5" markdown-include = "^0.8.1" -mkdocs = "^1.4.2" +mkdocs = "^1" ########################################################################### # LINTING CONFIGURATION diff --git a/quinn/transformations.py b/quinn/transformations.py index c0214e51..a2eac362 100644 --- a/quinn/transformations.py +++ b/quinn/transformations.py @@ -241,7 +241,7 @@ def fix_nullability(field: StructField, result_dict: dict) -> None: if spark is None: spark = SparkSession.builder.getOrCreate() - return spark.sparkSession.createDataFrame(output.rdd, output.schema) + return spark.createDataFrame(output.rdd, output.schema) def flatten_struct(df: DataFrame, col_name: str, separator: str = ":") -> DataFrame: From d9d6a4d06b4a719b0d800d85c913c6facc853cb8 Mon Sep 17 00:00:00 2001 From: SemyonSinchenko Date: Sat, 18 Nov 2023 22:17:46 +0000 Subject: [PATCH 8/9] Fix missing annotations import On branch feature/fix-tests Changes to be committed: modified: tests/test_transformations.py --- tests/test_transformations.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_transformations.py b/tests/test_transformations.py index 83c4d904..f9dd87de 100644 --- a/tests/test_transformations.py +++ b/tests/test_transformations.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import pytest import chispa import quinn @@ -848,4 +850,4 @@ def test_flatten_dataframe(spark): ) expected_df = spark.createDataFrame(expected_data, expected_schema) result_df = flatten_dataframe(df) - chispa.assert_df_equality(result_df, expected_df) \ No newline at end of file + chispa.assert_df_equality(result_df, expected_df) From 798e0b7244193ac8a07fba627305c710e710489c Mon Sep 17 00:00:00 2001 From: SemyonSinchenko Date: Sat, 18 Nov 2023 22:24:02 +0000 Subject: [PATCH 9/9] Fix spark 2.4 missing attribute On branch feature/fix-tests Changes to be committed: modified: quinn/transformations.py --- quinn/transformations.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/quinn/transformations.py b/quinn/transformations.py index a2eac362..6fc7f1f4 100644 --- a/quinn/transformations.py +++ b/quinn/transformations.py @@ -236,10 +236,11 @@ def fix_nullability(field: StructField, result_dict: dict) -> None: for field in output.schema: fix_nullability(field, result_dict) - spark = SparkSession.getActiveSession() - - if spark is None: + if not hasattr(SparkSession, "getActiveSession"): # spark 2.4 spark = SparkSession.builder.getOrCreate() + else: + spark = SparkSession.getActiveSession() + spark = spark if spark is not None else SparkSession.builder.getOrCreate() return spark.createDataFrame(output.rdd, output.schema)