From 1fdb5e4784677af17952f9fc0acca8793f25b80b Mon Sep 17 00:00:00 2001 From: Argenis Leon Date: Thu, 14 Nov 2019 00:37:00 -0600 Subject: [PATCH] Added levenshtein json test --- tests/creator/creator-profiler.ipynb | 72 +++++++++++++++----- tests/creator/creator.ipynb | 98 +++++++++++++++++++++------- tests/creator/creator.py | 6 ++ tests/test_df_distance_cluster.py | 4 ++ 4 files changed, 143 insertions(+), 37 deletions(-) diff --git a/tests/creator/creator-profiler.ipynb b/tests/creator/creator-profiler.ipynb index 963c606c..91b94560 100644 --- a/tests/creator/creator-profiler.ipynb +++ b/tests/creator/creator-profiler.ipynb @@ -41,10 +41,13 @@ "text": [ "C:\\Users\\argenisleon\\Anaconda3\\lib\\site-packages\\socks.py:58: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working\n", " from collections import Callable\n", + "../..\\optimus\\helpers\\functions.py:172: DeprecationWarning: invalid escape sequence \\d\n", + " pattern = '\\\"(\\d+\\.\\d+).*\\\"'\n", "\n", " You are using PySparkling of version 2.4.10, but your PySpark is of\n", " version 2.3.1. Please make sure Spark and PySparkling versions are compatible. \n", - "`formatargspec` is deprecated since Python 3.5. Use `signature` and the `Signature` object directly\n" + "`formatargspec` is deprecated since Python 3.5. Use `signature` and the `Signature` object directly\n", + "invalid escape sequence \\d\n" ] } ], @@ -69,7 +72,7 @@ "INFO:optimus:HADOOP_HOME=C:\\opt\\hadoop-2.7.7\n", "INFO:optimus:PYSPARK_PYTHON=C:\\Users\\argenisleon\\Anaconda3\\python.exe\n", "INFO:optimus:PYSPARK_DRIVER_PYTHON=jupyter\n", - "INFO:optimus:PYSPARK_SUBMIT_ARGS=--jars \"file:///C:/Users/argenisleon/Documents/Optimus/optimus/jars/RedshiftJDBC42-1.2.16.1027.jar,file:///C:/Users/argenisleon/Documents/Optimus/optimus/jars/mysql-connector-java-8.0.16.jar,file:///C:/Users/argenisleon/Documents/Optimus/optimus/jars/ojdbc8.jar,file:///C:/Users/argenisleon/Documents/Optimus/optimus/jars/postgresql-42.2.5.jar,file:///C:/Users/argenisleon/Documents/Optimus/optimus/jars/presto-jdbc-0.224.jar\" --driver-class-path \"C:/Users/argenisleon/Documents/Optimus/optimus/jars/RedshiftJDBC42-1.2.16.1027.jar;C:/Users/argenisleon/Documents/Optimus/optimus/jars/mysql-connector-java-8.0.16.jar;C:/Users/argenisleon/Documents/Optimus/optimus/jars/ojdbc8.jar;C:/Users/argenisleon/Documents/Optimus/optimus/jars/postgresql-42.2.5.jar;C:/Users/argenisleon/Documents/Optimus/optimus/jars/presto-jdbc-0.224.jar\" --conf \"spark.sql.catalogImplementation=hive\" pyspark-shell\n", + "INFO:optimus:PYSPARK_SUBMIT_ARGS=--jars \"file:///C:/Users/argenisleon/Documents/Optimus/optimus/jars/RedshiftJDBC42-1.2.16.1027.jar,file:///C:/Users/argenisleon/Documents/Optimus/optimus/jars/mysql-connector-java-8.0.16.jar,file:///C:/Users/argenisleon/Documents/Optimus/optimus/jars/ojdbc8.jar,file:///C:/Users/argenisleon/Documents/Optimus/optimus/jars/postgresql-42.2.5.jar,file:///C:/Users/argenisleon/Documents/Optimus/optimus/jars/presto-jdbc-0.224.jar,file:///C:/Users/argenisleon/Documents/Optimus/optimus/jars/spark-cassandra-connector_2.11-2.4.1.jar,file:///C:/Users/argenisleon/Documents/Optimus/optimus/jars/sqlite-jdbc-3.27.2.1.jar,file:///C:/Users/argenisleon/Documents/Optimus/optimus/jars/mssql-jdbc-7.4.1.jre8.jar\" --driver-class-path \"C:/Users/argenisleon/Documents/Optimus/optimus/jars/RedshiftJDBC42-1.2.16.1027.jar;C:/Users/argenisleon/Documents/Optimus/optimus/jars/mysql-connector-java-8.0.16.jar;C:/Users/argenisleon/Documents/Optimus/optimus/jars/ojdbc8.jar;C:/Users/argenisleon/Documents/Optimus/optimus/jars/postgresql-42.2.5.jar;C:/Users/argenisleon/Documents/Optimus/optimus/jars/presto-jdbc-0.224.jar;C:/Users/argenisleon/Documents/Optimus/optimus/jars/spark-cassandra-connector_2.11-2.4.1.jar;C:/Users/argenisleon/Documents/Optimus/optimus/jars/sqlite-jdbc-3.27.2.1.jar;C:/Users/argenisleon/Documents/Optimus/optimus/jars/mssql-jdbc-7.4.1.jre8.jar\" --conf \"spark.sql.catalogImplementation=hive\" pyspark-shell\n", "INFO:optimus:JAVA_HOME=C:\\java\n", "INFO:optimus:Pyarrow Installed\n", "INFO:optimus:-----\n", @@ -95,7 +98,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 5, "metadata": { "lines_to_next_cell": 2 }, @@ -1294,7 +1297,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -1303,7 +1306,7 @@ "\"a'a\"" ] }, - "execution_count": 28, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -1316,7 +1319,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -1333,7 +1336,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -1350,7 +1353,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -1372,7 +1375,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -5987,18 +5990,55 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating test_columns_agg() test function...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:optimus:test_columns_agg()\n", + "INFO:optimus:Batch Stats 0. Processing columns['names', 'height(ft)', 'function', 'rank', 'age', 'weight(t)', 'japanese name', 'last position seen', 'date arrival', 'last date seen', 'attributes', 'Date Type', 'timestamp', 'Cybertronian', 'function(binary)', 'NullType']\n", + "INFO:optimus:'kurtosis' function in 'age' column is returning 'nan'. Is that what you expected?. Seems that 'age' has 'nan' values\n", + "INFO:optimus:'skewness' function in 'age' column is returning 'nan'. Is that what you expected?. Seems that 'age' has 'nan' values\n", + "INFO:optimus:Batch Histogram 0. Processing columns['names', 'height(ft)', 'function', 'rank', 'age', 'weight(t)', 'japanese name', 'last position seen', 'date arrival', 'last date seen', 'attributes', 'Date Type', 'timestamp', 'Cybertronian', 'function(binary)', 'NullType']\n", + "INFO:optimus:'kurtosis' function in 'age' column is returning 'nan'. Is that what you expected?. Seems that 'age' has 'nan' values\n", + "INFO:optimus:'skewness' function in 'age' column is returning 'nan'. Is that what you expected?. Seems that 'age' has 'nan' values\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'names': {'count_uniques': 5, 'min': 'Jazz', 'max': 'ironhide&', 'count_na': 1, 'stddev': None, 'kurtosis': None, 'mean': None, 'skewness': None, 'sum': None, 'variance': None, 'zeros': 0}, 'height(ft)': {'count_uniques': 5, 'min': -28, 'max': 300, 'count_na': 2, 'stddev': 132.66612, 'kurtosis': 0.13863, 'mean': 65.6, 'skewness': 1.4049, 'sum': 328, 'variance': 17600.3, 'zeros': 0, 'percentile': {'0.75': 26, '0.95': 300, '0.05': -28, '0.25': 13, '0.5': 17}, 'hist': [{'count': 4.0, 'lower': -28.0, 'upper': 54.0}, {'count': 0.0, 'lower': 54.0, 'upper': 136.0}, {'count': 0.0, 'lower': 136.0, 'upper': 218.0}, {'count': 0.0, 'lower': 218.0, 'upper': 300.0}]}, 'function': {'count_uniques': 6, 'min': 'Battle Station', 'max': 'Security', 'count_na': 1, 'stddev': None, 'kurtosis': None, 'mean': None, 'skewness': None, 'sum': None, 'variance': None, 'zeros': 0}, 'rank': {'count_uniques': 3, 'min': 7, 'max': 10, 'count_na': 1, 'stddev': 1.36626, 'kurtosis': -1.5, 'mean': 8.33333, 'skewness': 0.3818, 'sum': 50, 'variance': 1.86667, 'zeros': 0, 'percentile': {'0.75': 10, '0.95': 10, '0.05': 7, '0.25': 7, '0.5': 8}, 'hist': [{'count': 4.0, 'lower': 7.0, 'upper': 8.5}, {'count': 0.0, 'lower': 8.5, 'upper': 10.0}]}, 'age': {'count_uniques': 1, 'min': 5000000, 'max': 5000000, 'count_na': 1, 'stddev': 0.0, 'kurtosis': nan, 'mean': 5000000.0, 'skewness': nan, 'sum': 30000000, 'variance': 0.0, 'zeros': 0, 'percentile': {'0.75': 5000000, '0.95': 5000000, '0.05': 5000000, '0.25': 5000000, '0.5': 5000000}, 'hist': [{'count': 6, 'lower': 5000000, 'upper': 5000001}]}, 'weight(t)': {'count_uniques': 5, 'min': 1.8, 'max': 5.7, 'count_na': 2, 'stddev': 1.64712, 'kurtosis': -1.43641, 'mean': 3.56, 'skewness': 0.06521, 'sum': 17.8, 'variance': 2.713, 'zeros': 0, 'percentile': {'0.75': 4.300000190734863, '0.95': 5.699999809265137, '0.05': 1.7999999523162842, '0.25': 2.0, '0.5': 4.0}, 'hist': [{'count': 1.0, 'lower': 1.8, 'upper': 2.78}, {'count': 0.0, 'lower': 2.78, 'upper': 3.75}, {'count': 2.0, 'lower': 3.75, 'upper': 4.73}, {'count': 1.0, 'lower': 4.73, 'upper': 5.7}]}, 'japanese name': {'count_uniques': 6, 'min': ['Bumble', 'Goldback'], 'max': ['Roadbuster'], 'count_na': 1}, 'last position seen': {'count_uniques': 4, 'min': '10.642707,-71.612534', 'max': '37.789563,-122.400356', 'count_na': 3, 'stddev': None, 'kurtosis': None, 'mean': None, 'skewness': None, 'sum': None, 'variance': None, 'zeros': 0}, 'date arrival': {'count_uniques': 1, 'min': '1980/04/10', 'max': '1980/04/10', 'count_na': 1, 'stddev': None, 'kurtosis': None, 'mean': None, 'skewness': None, 'sum': None, 'variance': None, 'zeros': 0}, 'last date seen': {'count_uniques': 6, 'min': '2011/04/10', 'max': '2016/09/10', 'count_na': 1, 'stddev': None, 'kurtosis': None, 'mean': None, 'skewness': None, 'sum': None, 'variance': None, 'zeros': 0}, 'attributes': {'count_uniques': 6, 'min': [None, 5700.0], 'max': [91.44000244140625, None], 'count_na': 1}, 'Date Type': {'count_uniques': 6, 'min': datetime.date(2011, 4, 10), 'max': datetime.date(2016, 9, 10), 'count_na': 1}, 'timestamp': {'count_uniques': 1, 'min': datetime.datetime(2014, 6, 24, 0, 0), 'max': datetime.datetime(2014, 6, 24, 0, 0), 'count_na': 1}, 'Cybertronian': {'count_uniques': 1, 'min': 1, 'max': 1, 'count_na': 1}, 'function(binary)': {'count_uniques': 6, 'min': bytearray(b'Battle Station'), 'max': bytearray(b'Security'), 'count_na': 1}, 'NullType': {'count_uniques': 0, 'min': None, 'max': None, 'count_na': 7}, 'p_count_na': 100.0, 'p_count_uniques': 0.0, 'range': 3.9000000000000004, 'median': 4.0, 'interquartile_range': 2.3000001907348633, 'coef_variation': 0.46267, 'mad': 1.7}\n" + ] + } + ], "source": [ "t.create(p, \"columns_agg\", None, 'json', None, source_df,\"*\")" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating file ../test_df_profiler.py\n", + "Done\n" + ] + } + ], "source": [ "t.run()" ] @@ -6006,10 +6046,12 @@ { "cell_type": "code", "execution_count": 39, - "metadata": {}, + "metadata": { + "lines_to_next_cell": 2 + }, "outputs": [], "source": [ - "a = \"{'name'=a'a}\"\n" + "a = \"{'name'=a'a}\"" ] }, { diff --git a/tests/creator/creator.ipynb b/tests/creator/creator.ipynb index 3307f0f5..7f5460f5 100644 --- a/tests/creator/creator.ipynb +++ b/tests/creator/creator.ipynb @@ -95,7 +95,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 5, "metadata": { "lines_to_next_cell": 2 }, @@ -1271,7 +1271,7 @@ }, { "cell_type": "code", - "execution_count": 128, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -102993,7 +102993,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -103002,7 +103002,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 9, "metadata": { "inputHidden": false, "outputHidden": false @@ -103021,7 +103021,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 10, "metadata": { "inputHidden": false, "outputHidden": false @@ -103099,17 +103099,17 @@ " \n", " \n", " \n", - "
\n", + "
\n", " \n", - " estadodemexico\n", + " distritofederal\n", " \n", "
\n", " \n", " \n", " \n", - "
\n", + "
\n", " \n", - " estadodemexico\n", + " distritofederal\n", " \n", "
\n", " \n", @@ -103127,17 +103127,17 @@ " \n", " \n", " \n", - "
\n", + "
\n", " \n", - " estadodemexico\n", + " distritofederal\n", " \n", "
\n", " \n", " \n", " \n", - "
\n", + "
\n", " \n", - " distritofederal\n", + " estadodemexico\n", " \n", "
\n", " \n", @@ -103155,17 +103155,17 @@ " \n", " \n", " \n", - "
\n", + "
\n", " \n", - " distritofederal\n", + " estadodemexico\n", " \n", "
\n", " \n", " \n", " \n", - "
\n", + "
\n", " \n", - " estadodemexico\n", + " distritofederal\n", " \n", "
\n", " \n", @@ -103183,17 +103183,17 @@ " \n", " \n", " \n", - "
\n", + "
\n", " \n", - " distritofederal\n", + " estadodemexico\n", " \n", "
\n", " \n", " \n", " \n", - "
\n", + "
\n", " \n", - " distritofederal\n", + " estadodemexico\n", " \n", "
\n", " \n", @@ -103227,6 +103227,24 @@ "t.create(dc, \"levenshtein_matrix\", None, 'df', None, source_df, \"STATE\")" ] }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating file ../test_df_distance_cluster.py\n", + "Done\n" + ] + } + ], + "source": [ + "t.run()" + ] + }, { "cell_type": "code", "execution_count": 9, @@ -103381,7 +103399,43 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating test_levenshtein_json() test function...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:optimus:test_levenshtein_json()\n", + "INFO:optimus:Using 'column_exp' to process column 'STATE***FINGERPRINT' with function _trim\n", + "INFO:optimus:Using 'column_exp' to process column 'STATE***FINGERPRINT' with function _lower\n", + "INFO:optimus:Using 'pandas_udf' to process column 'STATE***FINGERPRINT' with function multiple_replace\n", + "INFO:optimus:Using 'pandas_udf' to process column 'STATE***FINGERPRINT' with function _remove_accents\n", + "INFO:optimus:Using 'pandas_udf' to process column 'STATE***FINGERPRINT' with function _split_sort_remove_join\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'Estado de México': ['Distrito Federal'], 'Distrito Federal': ['Estado de México']}\n" + ] + } + ], + "source": [ + "t.create(dc, \"levenshtein_json\", None, 'dict', None, source_df, \"STATE\")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, "metadata": { "inputHidden": false, "lines_to_next_cell": 0, diff --git a/tests/creator/creator.py b/tests/creator/creator.py index b8c91c29..04b3497b 100644 --- a/tests/creator/creator.py +++ b/tests/creator/creator.py @@ -933,9 +933,15 @@ def func(col_name, attrs): # + {"outputHidden": false, "inputHidden": false} t.create(dc, "levenshtein_matrix", None, 'df', None, source_df, "STATE") +# - + +t.run() # + {"outputHidden": false, "inputHidden": false} t.create(dc, "levenshtein_filter", None, 'df', None, source_df, "STATE") +# - + +t.create(dc, "levenshtein_json", None, 'dict', None, source_df, "STATE") # + {"outputHidden": false, "inputHidden": false} t.run() diff --git a/tests/test_df_distance_cluster.py b/tests/test_df_distance_cluster.py index 159f0639..ebbc4bdf 100644 --- a/tests/test_df_distance_cluster.py +++ b/tests/test_df_distance_cluster.py @@ -18,6 +18,10 @@ def test_levenshtein_filter(): actual_df =dc.levenshtein_filter(source_df,'STATE') expected_df = op.create.df([('STATE_FROM', StringType(), True),('STATE***LEVENSHTEIN_DISTANCE', IntegerType(), True),('STATE_TO', StringType(), True)], [('estadodemexico', 11, 'distritofederal'), ('distritofederal', 11, 'estadodemexico')]) assert (expected_df.collect() == actual_df.collect()) + def test_levenshtein_json(self): + actual_df =dc.levenshtein_json(source_df,'STATE') + expected_value ={'Estado de México': ['Distrito Federal'], 'Distrito Federal': ['Estado de México']} + self.assertDictEqual(deep_sort(expected_value), deep_sort(actual_df)) @staticmethod def test_levenshtein_matrix(): actual_df =dc.levenshtein_matrix(source_df,'STATE')