From cee354e240d7017e90971ee30e0e2291fc2e7dda Mon Sep 17 00:00:00 2001 From: Nijanthan Vijayakumar <6072170+nijanthanvijayakumar@users.noreply.github.com> Date: Thu, 11 Jul 2024 20:52:54 +1000 Subject: [PATCH 1/5] Remove extension functions Related to #237 Remove deprecated extension functions from the codebase * Delete `quinn/extensions/dataframe_ext.py` and `quinn/extensions/spark_session_ext.py` * Remove import statements for `dataframe_ext` and `spark_session_ext` in `quinn/extensions/__init__.py` * Remove per-file ignores for `quinn/extensions/dataframe_ext.py` and `quinn/extensions/__init__.py` in `pyproject.toml` * Delete test files `tests/extensions/test_dataframe_ext.py` and `tests/extensions/test_spark_session_ext.py` --- pyproject.toml | 2 - quinn/extensions/__init__.py | 3 - quinn/extensions/dataframe_ext.py | 29 ----- quinn/extensions/spark_session_ext.py | 48 --------- tests/extensions/test_dataframe_ext.py | 118 --------------------- tests/extensions/test_spark_session_ext.py | 25 ----- 6 files changed, 225 deletions(-) delete mode 100644 quinn/extensions/dataframe_ext.py delete mode 100644 quinn/extensions/spark_session_ext.py delete mode 100644 tests/extensions/test_dataframe_ext.py delete mode 100644 tests/extensions/test_spark_session_ext.py diff --git a/pyproject.toml b/pyproject.toml index b0f5ed10..f3b4e51a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,8 +98,6 @@ ignore = [ extend-exclude = ["tests", "docs"] [tool.ruff.per-file-ignores] -"quinn/extensions/column_ext.py" = ["FBT003", "N802"] -"quinn/extensions/__init__.py" = ["F401", "F403"] "quinn/__init__.py" = ["F401", "F403"] "quinn/functions.py" = ["FBT003"] "quinn/keyword_finder.py" = ["A002"] diff --git a/quinn/extensions/__init__.py b/quinn/extensions/__init__.py index a78f76fd..7042ba11 100644 --- a/quinn/extensions/__init__.py +++ b/quinn/extensions/__init__.py @@ -12,6 +12,3 @@ # limitations under the License. """Extensions API.""" - -from quinn.extensions.dataframe_ext import * -from quinn.extensions.spark_session_ext import * diff --git a/quinn/extensions/dataframe_ext.py b/quinn/extensions/dataframe_ext.py deleted file mode 100644 index 049d4c0d..00000000 --- a/quinn/extensions/dataframe_ext.py +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import warnings - -from pyspark.sql import SparkSession -from pyspark.sql.dataframe import DataFrame - - -def _ext_function(spark: SparkSession, f: object) -> object: - warnings.warn( - "Extensions may be removed in the future versions of quinn. Please use explicit functions instead", - category=DeprecationWarning, - stacklevel=2, - ) - return f(spark) - - -DataFrame.transform = getattr(DataFrame, "transform", _ext_function) diff --git a/quinn/extensions/spark_session_ext.py b/quinn/extensions/spark_session_ext.py deleted file mode 100644 index f5f43544..00000000 --- a/quinn/extensions/spark_session_ext.py +++ /dev/null @@ -1,48 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -import warnings - -from pyspark.sql import DataFrame, SparkSession -from pyspark.sql.types import StructField, StructType - - -def create_df( - spark: SparkSession, - rows_data: list[tuple], - col_specs: list[tuple], -) -> DataFrame: - """Creates a new DataFrame from the given data and column specs. - - The returned DataFrame is created using the StructType and StructField classes provided by PySpark. - - :param rows_data: the data used to create the DataFrame - :type rows_data: array-like - :param col_specs: list of tuples containing the name and type of the field - :type col_specs: list of tuples - :return: a new DataFrame - :rtype: DataFrame - """ - warnings.warn( - "Extensions may be removed in the future versions of quinn. Please use `quinn.create_df()` instead", - category=DeprecationWarning, - stacklevel=2, - ) - - struct_fields = [StructField(*x) for x in col_specs] - return spark.createDataFrame(data=rows_data, schema=StructType(struct_fields)) - - -SparkSession.create_df = create_df diff --git a/tests/extensions/test_dataframe_ext.py b/tests/extensions/test_dataframe_ext.py deleted file mode 100644 index 189d03ce..00000000 --- a/tests/extensions/test_dataframe_ext.py +++ /dev/null @@ -1,118 +0,0 @@ -from functools import partial -import pytest -import pyspark -import chispa -from pyspark.sql.functions import col - -from ..spark import spark - -from .dataframe_transformations import ( - with_greeting, - with_something, - with_funny, - with_jacket, -) - - -def test_verbose_code_without_transform(): - data = [("jose", 1), ("li", 2), ("liz", 3)] - source_df = spark.createDataFrame(data, ["name", "age"]) - df1 = with_greeting(source_df) - df2 = with_something(df1, "moo") - expected_data = [ - ("jose", 1, "hi", "moo"), - ("li", 2, "hi", "moo"), - ("liz", 3, "hi", "moo"), - ] - expected_df = spark.createDataFrame( - expected_data, ["name", "age", "greeting", "something"] - ) - chispa.assert_df_equality(df2, expected_df, ignore_nullable=True) - - -@pytest.mark.skipif(pyspark.__version__ < '3.0', reason="df.transform not available for Spark<3.0") -def test_transform_with_lambda(): - data = [("jose", 1), ("li", 2), ("liz", 3)] - source_df = spark.createDataFrame(data, ["name", "age"]) - actual_df = source_df.transform( - lambda df: df.withColumn("age_times_two", col("age") * 2) - ) - expected_data = [("jose", 1, 2), ("li", 2, 4), ("liz", 3, 6)] - expected_df = spark.createDataFrame(expected_data, ["name", "age", "age_times_two"]) - chispa.assert_df_equality(actual_df, expected_df) - - -@pytest.mark.skipif(pyspark.__version__ < '3.0', reason="df.transform not available for Spark<3.0") -def test_transform_with_no_arg_fun(): - data = [("jose", 1), ("li", 2), ("liz", 3)] - source_df = spark.createDataFrame(data, ["name", "age"]) - actual_df = source_df.transform(lambda df: with_greeting(df)) - expected_data = [("jose", 1, "hi"), ("li", 2, "hi"), ("liz", 3, "hi")] - expected_df = spark.createDataFrame(expected_data, ["name", "age", "greeting"]) - chispa.assert_df_equality(actual_df, expected_df, ignore_nullable=True) - - -@pytest.mark.skipif(pyspark.__version__ < '3.0', reason="df.transform not available for Spark<3.0") -def test_transform_with_one_arg_fun(): - data = [("jose", 1), ("li", 2), ("liz", 3)] - source_df = spark.createDataFrame(data, ["name", "age"]) - actual_df = source_df.transform(lambda df: with_something(df, "crazy")) - expected_data = [("jose", 1, "crazy"), ("li", 2, "crazy"), ("liz", 3, "crazy")] - expected_df = spark.createDataFrame(expected_data, ["name", "age", "something"]) - chispa.assert_df_equality(actual_df, expected_df, ignore_nullable=True) - - -@pytest.mark.skipif(pyspark.__version__ < '3.0', reason="df.transform not available for Spark<3.0") -def test_chain_transforms(): - data = [("jose", 1), ("li", 2), ("liz", 3)] - source_df = spark.createDataFrame(data, ["name", "age"]) - actual_df = source_df.transform(with_greeting).transform( - lambda df: with_something(df, "crazy") - ) - expected_data = [ - ("jose", 1, "hi", "crazy"), - ("li", 2, "hi", "crazy"), - ("liz", 3, "hi", "crazy"), - ] - expected_df = spark.createDataFrame( - expected_data, ["name", "age", "greeting", "something"] - ) - chispa.assert_df_equality(actual_df, expected_df, ignore_nullable=True) - - -@pytest.mark.skipif(pyspark.__version__ < '3.0', reason="df.transform not available for Spark<3.0") -def test_transform_with_closure(): - data = [("jose", 1), ("li", 2), ("liz", 3)] - source_df = spark.createDataFrame(data, ["name", "age"]) - actual_df = source_df.transform(with_greeting).transform( # no lambda required - with_funny("haha") - ) - expected_data = [ - ("jose", 1, "hi", "haha"), - ("li", 2, "hi", "haha"), - ("liz", 3, "hi", "haha"), - ] - expected_df = spark.createDataFrame( - expected_data, ["name", "age", "greeting", "funny"] - ) - chispa.assert_df_equality(actual_df, expected_df, ignore_nullable=True) - - -@pytest.mark.skipif(pyspark.__version__ < '3.0', reason="df.transform not available for Spark<3.0") -def test_transform_with_functools_partial(): - data = [("jose", 1), ("li", 2), ("liz", 3)] - source_df = spark.createDataFrame(data, ["name", "age"]) - actual_df = source_df.transform( - partial(with_greeting) - ).transform( # partial is optional for transformations that only take a single DataFrame argument - partial(with_jacket, "warm") - ) - expected_data = [ - ("jose", 1, "hi", "warm"), - ("li", 2, "hi", "warm"), - ("liz", 3, "hi", "warm"), - ] - expected_df = spark.createDataFrame( - expected_data, ["name", "age", "greeting", "jacket"] - ) - chispa.assert_df_equality(actual_df, expected_df, ignore_nullable=True) diff --git a/tests/extensions/test_spark_session_ext.py b/tests/extensions/test_spark_session_ext.py deleted file mode 100644 index 301ccce6..00000000 --- a/tests/extensions/test_spark_session_ext.py +++ /dev/null @@ -1,25 +0,0 @@ -from pyspark.sql.types import StructType, StructField, StringType - -from ..spark import spark - -import chispa -import quinn - - -def test_create_df(): - schema = StructType( - [ - StructField("name", StringType(), True), - StructField("blah", StringType(), True), - ] - ) - data = [("jose", "a"), ("li", "b"), ("sam", "c")] - actual_df = spark.createDataFrame(data, schema) - - expected_df = quinn.create_df( - spark, - [("jose", "a"), ("li", "b"), ("sam", "c")], - [("name", StringType(), True), ("blah", StringType(), True)], - ) - - chispa.assert_df_equality(expected_df, actual_df) From 27526f1e3a2d5e228f4f677d114669903d8c33a2 Mon Sep 17 00:00:00 2001 From: Niju Vijayakumar Date: Thu, 11 Jul 2024 20:56:28 +1000 Subject: [PATCH 2/5] Remove the extensions directory Since the extensions functions have been removed, remove the corresponding package/directory and the `__init__.py` file too. --- quinn/extensions/__init__.py | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 quinn/extensions/__init__.py diff --git a/quinn/extensions/__init__.py b/quinn/extensions/__init__.py deleted file mode 100644 index 7042ba11..00000000 --- a/quinn/extensions/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Extensions API.""" From 97c5ce5487033512a3ed1383cb937cdae3b6186b Mon Sep 17 00:00:00 2001 From: Niju Vijayakumar Date: Thu, 11 Jul 2024 21:18:48 +1000 Subject: [PATCH 3/5] Remove the functions that have been removed in previous commits The following functions have been removed from the init file * print_athena_create_table * exists * forall --- quinn/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/quinn/__init__.py b/quinn/__init__.py index feca32c8..469cac82 100644 --- a/quinn/__init__.py +++ b/quinn/__init__.py @@ -17,7 +17,6 @@ from quinn.dataframe_helpers import ( column_to_list, create_df, - print_athena_create_table, show_output_to_df, to_list_of_dictionaries, two_columns_to_dictionary, @@ -35,8 +34,6 @@ approx_equal, array_choice, business_days_between, - exists, - forall, is_false, is_falsy, is_not_in, From 483b33481e000aac1b301b020a71b9cb9dd3ef09 Mon Sep 17 00:00:00 2001 From: Niju Vijayakumar Date: Fri, 12 Jul 2024 07:57:39 +1000 Subject: [PATCH 4/5] Restore SparkSession extension module & delete patching row 48 * Create the `extensions/` directory. * Create the `__init__.py` file within it with the license header. * Create the `spark_session_ext.py` file within the extensions directory * Update the doc-string in create_df to align with the parameter list. * Format modified files & lint the code using ruff (successful) --- pyproject.toml | 1 + quinn/extensions/__init__.py | 16 +++++++++ quinn/extensions/spark_session_ext.py | 48 +++++++++++++++++++++++++++ 3 files changed, 65 insertions(+) create mode 100644 quinn/extensions/__init__.py create mode 100644 quinn/extensions/spark_session_ext.py diff --git a/pyproject.toml b/pyproject.toml index f3b4e51a..cf84a6bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,6 +98,7 @@ ignore = [ extend-exclude = ["tests", "docs"] [tool.ruff.per-file-ignores] +"quinn/extensions/__init__.py" = ["F401", "F403"] "quinn/__init__.py" = ["F401", "F403"] "quinn/functions.py" = ["FBT003"] "quinn/keyword_finder.py" = ["A002"] diff --git a/quinn/extensions/__init__.py b/quinn/extensions/__init__.py new file mode 100644 index 00000000..1a86ff97 --- /dev/null +++ b/quinn/extensions/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Extensions API.""" + +from quinn.extensions.spark_session_ext import * diff --git a/quinn/extensions/spark_session_ext.py b/quinn/extensions/spark_session_ext.py new file mode 100644 index 00000000..db1aae8a --- /dev/null +++ b/quinn/extensions/spark_session_ext.py @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import warnings +from typing import TYPE_CHECKING + +from pyspark.sql.types import StructField, StructType + +if TYPE_CHECKING: + from pyspark.sql import DataFrame, SparkSession + + +def create_df( + spark: SparkSession, + rows_data: list[tuple], + col_specs: list[tuple], +) -> DataFrame: + """Creates a new DataFrame from the given data and column specifications. + + :param spark: SparkSession instance to create the DataFrame + :type spark: SparkSession + :param rows_data: The data used to populate the DataFrame, where each tuple represents a row. + :type rows_data: list[tuple] + :param col_specs: Specifications for the columns, where each tuple contains the column name and data type. + :type col_specs: list[tuple] + :return: A new DataFrame constructed from the provided rows and column specifications. + :rtype: DataFrame + """ + warnings.warn( + "Extensions may be removed in the future versions of quinn. Please use `quinn.create_df()` instead", + category=DeprecationWarning, + stacklevel=2, + ) + + struct_fields = [StructField(*x) for x in col_specs] + return spark.createDataFrame(data=rows_data, schema=StructType(struct_fields)) From d3d497abacdee9bc050c282a871b85a976f2cacc Mon Sep 17 00:00:00 2001 From: Niju Vijayakumar Date: Sun, 14 Jul 2024 18:33:44 +1000 Subject: [PATCH 5/5] Move the create_df function to dataframe_helpers As a matter of fact, a create_df function already exists under the dataframe_helpers.py Following are the changes introduced by this commit. * Update `dataframe_helpers.py` by updating the doc-string in the create_df function. * Remove quinn/extensions/ directory along with the two `.py` files involved. * Remove the ruff check from the pyproject.toml. * Finally, format the dataframe_helpers.py with ruff --- pyproject.toml | 1 - quinn/dataframe_helpers.py | 32 +++++++++--------- quinn/extensions/__init__.py | 16 --------- quinn/extensions/spark_session_ext.py | 48 --------------------------- 4 files changed, 16 insertions(+), 81 deletions(-) delete mode 100644 quinn/extensions/__init__.py delete mode 100644 quinn/extensions/spark_session_ext.py diff --git a/pyproject.toml b/pyproject.toml index cf84a6bc..f3b4e51a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,7 +98,6 @@ ignore = [ extend-exclude = ["tests", "docs"] [tool.ruff.per-file-ignores] -"quinn/extensions/__init__.py" = ["F401", "F403"] "quinn/__init__.py" = ["F401", "F403"] "quinn/functions.py" = ["FBT003"] "quinn/keyword_finder.py" = ["A002"] diff --git a/quinn/dataframe_helpers.py b/quinn/dataframe_helpers.py index c69af269..750c0580 100644 --- a/quinn/dataframe_helpers.py +++ b/quinn/dataframe_helpers.py @@ -45,10 +45,10 @@ def column_to_list(df: DataFrame, col_name: str) -> list[Any]: return [row[0] for row in df.select(col_name).collect()] pyarrow_enabled = ( - spark_session.conf.get( - "spark.sql.execution.arrow.pyspark.enabled", - ) - == "true" + spark_session.conf.get( + "spark.sql.execution.arrow.pyspark.enabled", + ) + == "true" ) pyarrow_valid = pyarrow_enabled and sys.modules["pyarrow"].__version__ >= "0.17.0" @@ -63,9 +63,9 @@ def column_to_list(df: DataFrame, col_name: str) -> list[Any]: def two_columns_to_dictionary( - df: DataFrame, - key_col_name: str, - value_col_name: str, + df: DataFrame, + key_col_name: str, + value_col_name: str, ) -> dict[str, Any]: """Collect two columns as dictionary when first column is key and second is value. @@ -114,18 +114,18 @@ def show_output_to_df(show_output: str, spark: SparkSession) -> DataFrame: return spark.createDataFrame(pretty_data, pretty_column_names) -def create_df(spark: SparkSession, rows_data, col_specs) -> DataFrame: # noqa: ANN001 - """Create a new DataFrame from the given data and column specs. +def create_df(spark: SparkSession, rows_data: list[tuple], col_specs: list[tuple]) -> DataFrame: + """Creates a new DataFrame from the given data and column specifications. - The returned DataFrame s created using the StructType and StructField classes provided by PySpark. + The returned DataFrame created using the StructType and StructField classes provided by PySpark. - :param spark: SparkSession object + :param spark: SparkSession object to create the DataFrame :type spark: SparkSession - :param rows_data: the data used to create the DataFrame - :type rows_data: array-like - :param col_specs: list of tuples containing the name and type of the field - :type col_specs: list of tuples - :return: a new DataFrame + :param rows_data: The data used to populate the DataFrame, where each tuple represents a row. + :type rows_data: list[tuple] + :param col_specs: list of tuples containing the name and type of the field, i.e., specifications for the columns. + :type col_specs: list[tuple] + :return: A new DataFrame constructed from the provided rows and column specifications. :rtype: DataFrame """ struct_fields = list(map(lambda x: StructField(*x), col_specs)) # noqa: C417 diff --git a/quinn/extensions/__init__.py b/quinn/extensions/__init__.py deleted file mode 100644 index 1a86ff97..00000000 --- a/quinn/extensions/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Extensions API.""" - -from quinn.extensions.spark_session_ext import * diff --git a/quinn/extensions/spark_session_ext.py b/quinn/extensions/spark_session_ext.py deleted file mode 100644 index db1aae8a..00000000 --- a/quinn/extensions/spark_session_ext.py +++ /dev/null @@ -1,48 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -import warnings -from typing import TYPE_CHECKING - -from pyspark.sql.types import StructField, StructType - -if TYPE_CHECKING: - from pyspark.sql import DataFrame, SparkSession - - -def create_df( - spark: SparkSession, - rows_data: list[tuple], - col_specs: list[tuple], -) -> DataFrame: - """Creates a new DataFrame from the given data and column specifications. - - :param spark: SparkSession instance to create the DataFrame - :type spark: SparkSession - :param rows_data: The data used to populate the DataFrame, where each tuple represents a row. - :type rows_data: list[tuple] - :param col_specs: Specifications for the columns, where each tuple contains the column name and data type. - :type col_specs: list[tuple] - :return: A new DataFrame constructed from the provided rows and column specifications. - :rtype: DataFrame - """ - warnings.warn( - "Extensions may be removed in the future versions of quinn. Please use `quinn.create_df()` instead", - category=DeprecationWarning, - stacklevel=2, - ) - - struct_fields = [StructField(*x) for x in col_specs] - return spark.createDataFrame(data=rows_data, schema=StructType(struct_fields))