diff --git a/spark_pipelines/ExamplePipelines.ipynb b/spark_pipelines/ExamplePipelines.ipynb index f0f9b77..9d286ad 100644 --- a/spark_pipelines/ExamplePipelines.ipynb +++ b/spark_pipelines/ExamplePipelines.ipynb @@ -10,7 +10,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 28, "id": "752eb795", "metadata": {}, "outputs": [], @@ -19,7 +19,7 @@ "import pandas as pd\n", "from pathlib import Path\n", "from sklearn.pipeline import make_pipeline\n", - "\n", + "import time\n", "from generators import *\n", "from clean_data import *\n", "from preprocess_data import *\n", @@ -50,6 +50,7 @@ { "cell_type": "code", "execution_count": 3, + "id": "e915186b", "metadata": {}, "outputs": [], "source": [ @@ -71,6 +72,7 @@ { "cell_type": "code", "execution_count": 5, + "id": "e9588a02", "metadata": {}, "outputs": [ { @@ -2432,6 +2434,48 @@ "## 2.1 Generating Features" ] }, + { + "cell_type": "code", + "execution_count": 31, + "id": "0899db27", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "60.85968780517578\n" + ] + } + ], + "source": [ + "start=time.time()\n", + "val=getAutoCovariance(df_prepared.reset_index()[:1000], 1, 10,\"volume\")\n", + "end=time.time()\n", + "print(end-start)" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "48a90b44", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.1977696418762207\n" + ] + } + ], + "source": [ + "start=time.time()\n", + "val=getAutoCovariance2(df_prepared.reset_index()[:1000], 1, 10,\"volume\")\n", + "end=time.time()\n", + "print(end-start)" + ] + }, { "attachments": {}, "cell_type": "markdown",