diff --git a/optimus/dataframe/rows.py b/optimus/dataframe/rows.py index ff91d0f7..4a94c922 100644 --- a/optimus/dataframe/rows.py +++ b/optimus/dataframe/rows.py @@ -6,7 +6,7 @@ from pyspark.sql import functions as F # Helpers -import optimus.optimus as op +from optimus import Optimus as op from infer import is_list_of_str_or_int, is_list_of_dataframes, is_list_of_tuples, is_dataframe, \ PYSPARK_NUMERIC_TYPES from optimus.audf import filter_row_by_data_type as fbdt diff --git a/tests/creator/creator.ipynb b/tests/creator/creator.ipynb index 35da143b..20d0719e 100644 --- a/tests/creator/creator.ipynb +++ b/tests/creator/creator.ipynb @@ -9,7 +9,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -19,7 +19,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": { "inputHidden": false, "outputHidden": false @@ -32,7 +32,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -52,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -105,7 +105,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 5, "metadata": { "lines_to_next_cell": 2 }, @@ -1281,7 +1281,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -12119,7 +12119,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -12141,7 +12141,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -12156,7 +12156,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -12166,7 +12166,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -12191,18 +12191,358 @@ ] }, { - "ename": "AttributeError", - "evalue": "module 'optimus' has no attribute 'Create'", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mAttributeError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcreate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"rows.append\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"df\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrow\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[1;32m~\\Documents\\Optimus\\optimus\\helpers\\test.py\u001b[0m in \u001b[0;36mcreate\u001b[1;34m(self, obj, method, suffix, output, additional_method, *args, **kwargs)\u001b[0m\n\u001b[0;32m 217\u001b[0m \u001b[0mdf_func\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf_func\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 218\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 219\u001b[1;33m \u001b[0mdf_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdf_func\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 220\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 221\u001b[0m \u001b[1;31m# Additional Methods\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32m~\\Documents\\Optimus\\optimus\\helpers\\decorators.py\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m 47\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 48\u001b[0m \u001b[0mstart_time\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtimeit\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdefault_timer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 49\u001b[1;33m \u001b[0mf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 50\u001b[0m \u001b[0m_time\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mround\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeit\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdefault_timer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mstart_time\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 51\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mlog_time\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;32m~\\Documents\\Optimus\\optimus\\dataframe\\rows.py\u001b[0m in \u001b[0;36mappend\u001b[1;34m(rows)\u001b[0m\n\u001b[0;32m 44\u001b[0m \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mis_list_of_tuples\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrows\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 45\u001b[0m \u001b[0mrows\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mtuple\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrows\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 46\u001b[1;33m \u001b[0mnew_row\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mop\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mCreate\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrows\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 47\u001b[0m \u001b[0mdf_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munion\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnew_row\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 48\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", - "\u001b[1;31mAttributeError\u001b[0m: module 'optimus' has no attribute 'Create'" - ] + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "
Viewing 5 of 5 rows / 6 columns
\n", + "
2 partition(s)
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
\n", + "
words
\n", + "
1 (string)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + "
\n", + "
num
\n", + "
2 (int)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + "
\n", + "
animals
\n", + "
3 (string)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + "
\n", + "
thing
\n", + "
4 (string)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + "
\n", + "
second
\n", + "
5 (int)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + "
\n", + "
filter
\n", + "
6 (string)
\n", + "
\n", + " \n", + " nullable\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " ⋅⋅I⋅like⋅⋅⋅⋅⋅fish⋅⋅\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " 1\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " dog⋅dog\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " housé\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " 5\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " a\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " ⋅⋅⋅⋅zombies\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " 2\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " cat\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " tv\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " 6\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " b\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " simpsons⋅⋅⋅cat⋅lady\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " 2\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " frog\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " table\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " 7\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " 1\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " None\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " 3\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " eagle\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " glass\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " 8\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " c\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " this⋅is⋅a⋅word\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " 2\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " this⋅is⋅an⋅animal\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " this⋅is⋅a⋅thing\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " 64\n", + " \n", + "
\n", + "
\n", + "
\n", + " \n", + " this⋅is⋅a⋅filter\n", + " \n", + "
\n", + "
\n", + "\n", + "\n", + "
Viewing 5 of 5 rows / 6 columns
\n", + "
2 partition(s)
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [