13_NY_Taxi_full_DuckDB_Polars.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uzuQ8idWUxa_"
   },
   "source": [
    "# NYC Taxi full dataset ~40 GB - BigData with DuckDB and Polars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "jcTNP9AIQMNb"
   },
   "outputs": [],
   "source": [
    "#https://github.com/gerardrbentley/uber-nyc-pickups-duckdb/blob/main/01_duck_streamlit.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "oM48uqj5Jmnb"
   },
   "outputs": [],
   "source": [
    "!pip install boto3 -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "iMTRolD1JeDt",
    "outputId": "3b0d2749-454e-4b15-f87c-ad6301822aa9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BEGIN DOWNLOAD!\n",
      "downloading nyc-taxi/2009/01/data.parquet\n",
      "downloaded nyc-taxi/2009/01/data.parquet\n",
      "downloading nyc-taxi/2009/02/data.parquet\n",
      "downloaded nyc-taxi/2009/02/data.parquet\n",
      "downloading nyc-taxi/2009/03/data.parquet\n",
      "downloaded nyc-taxi/2009/03/data.parquet\n",
      "downloading nyc-taxi/2009/04/data.parquet\n",
      "downloaded nyc-taxi/2009/04/data.parquet\n",
      "downloading nyc-taxi/2009/05/data.parquet\n",
      "downloaded nyc-taxi/2009/05/data.parquet\n",
      "downloading nyc-taxi/2009/06/data.parquet\n",
      "downloaded nyc-taxi/2009/06/data.parquet\n",
      "downloading nyc-taxi/2009/07/data.parquet\n",
      "downloaded nyc-taxi/2009/07/data.parquet\n",
      "downloading nyc-taxi/2009/08/data.parquet\n",
      "downloaded nyc-taxi/2009/08/data.parquet\n",
      "downloading nyc-taxi/2009/09/data.parquet\n",
      "downloaded nyc-taxi/2009/09/data.parquet\n",
      "downloading nyc-taxi/2009/10/data.parquet\n",
      "downloaded nyc-taxi/2009/10/data.parquet\n",
      "downloading nyc-taxi/2009/11/data.parquet\n",
      "downloaded nyc-taxi/2009/11/data.parquet\n",
      "downloading nyc-taxi/2009/12/data.parquet\n",
      "downloaded nyc-taxi/2009/12/data.parquet\n",
      "downloading nyc-taxi/2010/01/data.parquet\n",
      "downloaded nyc-taxi/2010/01/data.parquet\n",
      "downloading nyc-taxi/2010/02/data.parquet\n",
      "downloaded nyc-taxi/2010/02/data.parquet\n",
      "downloading nyc-taxi/2010/04/data.parquet\n",
      "downloaded nyc-taxi/2010/04/data.parquet\n",
      "downloading nyc-taxi/2010/05/data.parquet\n",
      "downloaded nyc-taxi/2010/05/data.parquet\n",
      "downloading nyc-taxi/2010/06/data.parquet\n",
      "downloaded nyc-taxi/2010/06/data.parquet\n",
      "downloading nyc-taxi/2010/07/data.parquet\n",
      "downloaded nyc-taxi/2010/07/data.parquet\n",
      "downloading nyc-taxi/2010/08/data.parquet\n",
      "downloaded nyc-taxi/2010/08/data.parquet\n",
      "downloading nyc-taxi/2010/09/data.parquet\n",
      "downloaded nyc-taxi/2010/09/data.parquet\n",
      "downloading nyc-taxi/2010/10/data.parquet\n",
      "downloaded nyc-taxi/2010/10/data.parquet\n",
      "downloading nyc-taxi/2010/11/data.parquet\n",
      "downloaded nyc-taxi/2010/11/data.parquet\n",
      "downloading nyc-taxi/2010/12/data.parquet\n",
      "downloaded nyc-taxi/2010/12/data.parquet\n",
      "downloading nyc-taxi/2011/01/data.parquet\n",
      "downloaded nyc-taxi/2011/01/data.parquet\n",
      "downloading nyc-taxi/2011/02/data.parquet\n",
      "downloaded nyc-taxi/2011/02/data.parquet\n",
      "downloading nyc-taxi/2011/03/data.parquet\n",
      "downloaded nyc-taxi/2011/03/data.parquet\n",
      "downloading nyc-taxi/2011/04/data.parquet\n",
      "downloaded nyc-taxi/2011/04/data.parquet\n",
      "downloading nyc-taxi/2011/05/data.parquet\n",
      "downloaded nyc-taxi/2011/05/data.parquet\n",
      "downloading nyc-taxi/2011/06/data.parquet\n",
      "downloaded nyc-taxi/2011/06/data.parquet\n",
      "downloading nyc-taxi/2011/07/data.parquet\n",
      "downloaded nyc-taxi/2011/07/data.parquet\n",
      "downloading nyc-taxi/2011/08/data.parquet\n",
      "downloaded nyc-taxi/2011/08/data.parquet\n",
      "downloading nyc-taxi/2011/09/data.parquet\n",
      "downloaded nyc-taxi/2011/09/data.parquet\n",
      "downloading nyc-taxi/2011/10/data.parquet\n",
      "downloaded nyc-taxi/2011/10/data.parquet\n",
      "downloading nyc-taxi/2011/11/data.parquet\n",
      "downloaded nyc-taxi/2011/11/data.parquet\n",
      "downloading nyc-taxi/2011/12/data.parquet\n",
      "downloaded nyc-taxi/2011/12/data.parquet\n",
      "downloading nyc-taxi/2012/01/data.parquet\n",
      "downloaded nyc-taxi/2012/01/data.parquet\n",
      "downloading nyc-taxi/2012/02/data.parquet\n",
      "downloaded nyc-taxi/2012/02/data.parquet\n",
      "downloading nyc-taxi/2012/03/data.parquet\n",
      "downloaded nyc-taxi/2012/03/data.parquet\n",
      "downloading nyc-taxi/2012/04/data.parquet\n",
      "downloaded nyc-taxi/2012/04/data.parquet\n",
      "downloading nyc-taxi/2012/05/data.parquet\n",
      "downloaded nyc-taxi/2012/05/data.parquet\n",
      "downloading nyc-taxi/2012/06/data.parquet\n",
      "downloaded nyc-taxi/2012/06/data.parquet\n",
      "downloading nyc-taxi/2012/07/data.parquet\n",
      "downloaded nyc-taxi/2012/07/data.parquet\n",
      "downloading nyc-taxi/2012/08/data.parquet\n",
      "downloaded nyc-taxi/2012/08/data.parquet\n",
      "downloading nyc-taxi/2012/09/data.parquet\n",
      "downloaded nyc-taxi/2012/09/data.parquet\n",
      "downloading nyc-taxi/2012/10/data.parquet\n",
      "downloaded nyc-taxi/2012/10/data.parquet\n",
      "downloading nyc-taxi/2012/11/data.parquet\n",
      "downloaded nyc-taxi/2012/11/data.parquet\n",
      "downloading nyc-taxi/2012/12/data.parquet\n",
      "downloaded nyc-taxi/2012/12/data.parquet\n",
      "downloading nyc-taxi/2013/01/data.parquet\n",
      "downloaded nyc-taxi/2013/01/data.parquet\n",
      "downloading nyc-taxi/2013/02/data.parquet\n",
      "downloaded nyc-taxi/2013/02/data.parquet\n",
      "downloading nyc-taxi/2013/03/data.parquet\n",
      "downloaded nyc-taxi/2013/03/data.parquet\n",
      "downloading nyc-taxi/2013/04/data.parquet\n",
      "downloaded nyc-taxi/2013/04/data.parquet\n",
      "downloading nyc-taxi/2013/05/data.parquet\n",
      "downloaded nyc-taxi/2013/05/data.parquet\n",
      "downloading nyc-taxi/2013/06/data.parquet\n",
      "downloaded nyc-taxi/2013/06/data.parquet\n",
      "downloading nyc-taxi/2013/07/data.parquet\n",
      "downloaded nyc-taxi/2013/07/data.parquet\n",
      "downloading nyc-taxi/2013/08/data.parquet\n",
      "downloaded nyc-taxi/2013/08/data.parquet\n",
      "downloading nyc-taxi/2013/09/data.parquet\n",
      "downloaded nyc-taxi/2013/09/data.parquet\n",
      "downloading nyc-taxi/2013/10/data.parquet\n",
      "downloaded nyc-taxi/2013/10/data.parquet\n",
      "downloading nyc-taxi/2013/11/data.parquet\n",
      "downloaded nyc-taxi/2013/12/data.parquet\n",
      "downloading nyc-taxi/2014/01/data.parquet\n",
      "downloaded nyc-taxi/2014/01/data.parquet\n",
      "downloading nyc-taxi/2014/02/data.parquet\n",
      "downloaded nyc-taxi/2014/02/data.parquet\n",
      "downloading nyc-taxi/2014/03/data.parquet\n",
      "downloaded nyc-taxi/2014/03/data.parquet\n",
      "downloading nyc-taxi/2014/04/data.parquet\n",
      "downloaded nyc-taxi/2014/04/data.parquet\n",
      "downloading nyc-taxi/2014/05/data.parquet\n",
      "downloaded nyc-taxi/2014/05/data.parquet\n",
      "downloading nyc-taxi/2014/06/data.parquet\n",
      "downloaded nyc-taxi/2014/06/data.parquet\n",
      "downloading nyc-taxi/2014/07/data.parquet\n",
      "downloaded nyc-taxi/2014/07/data.parquet\n",
      "downloading nyc-taxi/2014/08/data.parquet\n",
      "downloaded nyc-taxi/2014/08/data.parquet\n",
      "downloading nyc-taxi/2014/09/data.parquet\n",
      "downloaded nyc-taxi/2014/09/data.parquet\n",
      "downloading nyc-taxi/2014/10/data.parquet\n",
      "downloaded nyc-taxi/2014/10/data.parquet\n",
      "downloading nyc-taxi/2014/11/data.parquet\n",
      "downloaded nyc-taxi/2014/11/data.parquet\n",
      "downloading nyc-taxi/2014/12/data.parquet\n",
      "downloaded nyc-taxi/2014/12/data.parquet\n",
      "downloading nyc-taxi/2015/01/data.parquet\n",
      "downloaded nyc-taxi/2015/01/data.parquet\n",
      "downloading nyc-taxi/2015/02/data.parquet\n",
      "downloaded nyc-taxi/2015/02/data.parquet\n",
      "downloading nyc-taxi/2015/03/data.parquet\n",
      "downloaded nyc-taxi/2015/03/data.parquet\n",
      "downloading nyc-taxi/2015/04/data.parquet\n",
      "downloaded nyc-taxi/2015/04/data.parquet\n",
      "downloading nyc-taxi/2015/05/data.parquet\n",
      "downloaded nyc-taxi/2015/05/data.parquet\n",
      "downloading nyc-taxi/2015/06/data.parquet\n",
      "downloaded nyc-taxi/2015/06/data.parquet\n",
      "downloading nyc-taxi/2015/07/data.parquet\n",
      "downloaded nyc-taxi/2015/07/data.parquet\n",
      "downloading nyc-taxi/2015/08/data.parquet\n",
      "downloaded nyc-taxi/2015/08/data.parquet\n",
      "downloading nyc-taxi/2015/09/data.parquet\n",
      "downloaded nyc-taxi/2015/09/data.parquet\n",
      "downloading nyc-taxi/2015/10/data.parquet\n",
      "downloaded nyc-taxi/2015/10/data.parquet\n",
      "downloading nyc-taxi/2015/11/data.parquet\n",
      "downloaded nyc-taxi/2015/11/data.parquet\n",
      "downloading nyc-taxi/2015/12/data.parquet\n",
      "downloaded nyc-taxi/2015/12/data.parquet\n",
      "downloading nyc-taxi/2016/01/data.parquet\n",
      "downloaded nyc-taxi/2016/01/data.parquet\n",
      "downloading nyc-taxi/2016/02/data.parquet\n",
      "downloaded nyc-taxi/2016/02/data.parquet\n",
      "downloading nyc-taxi/2016/03/data.parquet\n",
      "downloaded nyc-taxi/2016/03/data.parquet\n",
      "downloading nyc-taxi/2016/04/data.parquet\n",
      "downloaded nyc-taxi/2016/04/data.parquet\n",
      "downloading nyc-taxi/2016/05/data.parquet\n",
      "downloaded nyc-taxi/2016/05/data.parquet\n",
      "downloading nyc-taxi/2016/06/data.parquet\n",
      "downloaded nyc-taxi/2016/06/data.parquet\n",
      "downloading nyc-taxi/2016/07/data.parquet\n",
      "downloaded nyc-taxi/2016/07/data.parquet\n",
      "downloading nyc-taxi/2016/08/data.parquet\n",
      "downloaded nyc-taxi/2016/08/data.parquet\n",
      "downloading nyc-taxi/2016/09/data.parquet\n",
      "downloaded nyc-taxi/2016/09/data.parquet\n",
      "downloading nyc-taxi/2016/10/data.parquet\n",
      "downloaded nyc-taxi/2016/10/data.parquet\n",
      "downloading nyc-taxi/2016/11/data.parquet\n",
      "downloaded nyc-taxi/2016/11/data.parquet\n",
      "downloading nyc-taxi/2016/12/data.parquet\n",
      "downloaded nyc-taxi/2016/12/data.parquet\n",
      "downloading nyc-taxi/2017/01/data.parquet\n",
      "downloaded nyc-taxi/2017/01/data.parquet\n",
      "downloading nyc-taxi/2017/02/data.parquet\n",
      "downloaded nyc-taxi/2017/02/data.parquet\n",
      "downloading nyc-taxi/2017/03/data.parquet\n",
      "downloaded nyc-taxi/2017/03/data.parquet\n",
      "downloading nyc-taxi/2017/04/data.parquet\n",
      "downloaded nyc-taxi/2017/04/data.parquet\n",
      "downloading nyc-taxi/2017/05/data.parquet\n",
      "downloaded nyc-taxi/2017/05/data.parquet\n",
      "downloading nyc-taxi/2017/06/data.parquet\n",
      "downloaded nyc-taxi/2017/06/data.parquet\n",
      "downloading nyc-taxi/2017/07/data.parquet\n",
      "downloaded nyc-taxi/2017/07/data.parquet\n",
      "downloading nyc-taxi/2017/08/data.parquet\n",
      "downloaded nyc-taxi/2017/08/data.parquet\n",
      "downloading nyc-taxi/2017/09/data.parquet\n",
      "downloaded nyc-taxi/2017/09/data.parquet\n",
      "downloading nyc-taxi/2017/10/data.parquet\n",
      "downloaded nyc-taxi/2017/10/data.parquet\n",
      "downloading nyc-taxi/2017/11/data.parquet\n",
      "downloaded nyc-taxi/2017/11/data.parquet\n",
      "downloading nyc-taxi/2017/12/data.parquet\n",
      "downloaded nyc-taxi/2017/12/data.parquet\n",
      "downloading nyc-taxi/2018/01/data.parquet\n",
      "downloaded nyc-taxi/2018/01/data.parquet\n",
      "downloading nyc-taxi/2018/02/data.parquet\n",
      "downloaded nyc-taxi/2018/02/data.parquet\n",
      "downloading nyc-taxi/2018/03/data.parquet\n",
      "downloaded nyc-taxi/2018/03/data.parquet\n",
      "downloading nyc-taxi/2018/04/data.parquet\n",
      "downloaded nyc-taxi/2018/04/data.parquet\n",
      "downloading nyc-taxi/2018/05/data.parquet\n",
      "downloaded nyc-taxi/2018/05/data.parquet\n",
      "downloading nyc-taxi/2018/06/data.parquet\n",
      "downloaded nyc-taxi/2018/06/data.parquet\n",
      "downloading nyc-taxi/2018/07/data.parquet\n",
      "downloaded nyc-taxi/2018/07/data.parquet\n",
      "downloading nyc-taxi/2018/08/data.parquet\n",
      "downloaded nyc-taxi/2018/08/data.parquet\n",
      "downloading nyc-taxi/2018/09/data.parquet\n",
      "downloaded nyc-taxi/2018/09/data.parquet\n",
      "downloading nyc-taxi/2018/10/data.parquet\n",
      "downloaded nyc-taxi/2018/10/data.parquet\n",
      "downloading nyc-taxi/2018/11/data.parquet\n",
      "downloaded nyc-taxi/2018/11/data.parquet\n",
      "downloading nyc-taxi/2018/12/data.parquet\n",
      "downloaded nyc-taxi/2018/12/data.parquet\n",
      "downloading nyc-taxi/2019/01/data.parquet\n",
      "downloaded nyc-taxi/2019/01/data.parquet\n",
      "downloading nyc-taxi/2019/02/data.parquet\n",
      "downloaded nyc-taxi/2019/02/data.parquet\n",
      "downloading nyc-taxi/2019/03/data.parquet\n",
      "downloaded nyc-taxi/2019/03/data.parquet\n",
      "downloading nyc-taxi/2019/04/data.parquet\n",
      "downloaded nyc-taxi/2019/04/data.parquet\n",
      "downloading nyc-taxi/2019/05/data.parquet\n",
      "downloaded nyc-taxi/2019/05/data.parquet\n",
      "downloading nyc-taxi/2019/06/data.parquet\n",
      "downloaded nyc-taxi/2019/06/data.parquet\n",
      "DONE DOWNLOAD!!!\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "import boto3\n",
    "import os\n",
    "from botocore import UNSIGNED\n",
    "from botocore.client import Config\n",
    "from botocore.exceptions import ResponseStreamingError\n",
    "\n",
    "s3 = boto3.resource(\"s3\", config=Config(signature_version=UNSIGNED))\n",
    "\n",
    "\n",
    "def download_s3_folder(bucket_name, s3_folder, local_dir=None):\n",
    "    bucket = s3.Bucket(bucket_name)\n",
    "    for obj in bucket.objects.filter(Prefix=s3_folder):\n",
    "        target = (\n",
    "            obj.key\n",
    "            if local_dir is None\n",
    "            else os.path.join(local_dir, os.path.relpath(obj.key, s3_folder))\n",
    "        )\n",
    "        if os.path.exists(target):\n",
    "            print(f\"skipping {target}\")\n",
    "            continue\n",
    "        if not os.path.exists(os.path.dirname(target)):\n",
    "            os.makedirs(os.path.dirname(target))\n",
    "        if obj.key[-1] == \"/\":\n",
    "            continue\n",
    "        print(f\"downloading {target}\")\n",
    "        try:\n",
    "            bucket.download_file(obj.key, target)\n",
    "        except ResponseStreamingError:\n",
    "            time.sleep(60)\n",
    "            bucket.download_file(obj.key, target)\n",
    "        print(f\"downloaded {target}\")\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    print(\"BEGIN DOWNLOAD!\")\n",
    "    download_s3_folder(\"ursa-labs-taxi-data\", \"\", \"nyc-taxi\")\n",
    "    print(\"DONE DOWNLOAD!!!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000,
     "referenced_widgets": [
      "1cde922d422649f88db7a9974ccaf6e8",
      "15c42a076524404589e0b295e2f49198",
      "ade41a377f7c4ea2876540f9f169047c"
     ]
    },
    "id": "3X6megKnJd-F",
    "outputId": "e26a2f10-b764-48e1-d3c5-416554ca8db6"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1cde922d422649f88db7a9974ccaf6e8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "chunk total 0\n",
      "chunk total 1000000\n",
      "chunk total 2000000\n",
      "chunk total 3000000\n",
      "chunk total 4000000\n",
      "chunk total 5000000\n",
      "chunk total 6000000\n",
      "chunk total 7000000\n",
      "chunk total 8000000\n",
      "chunk total 9000000\n",
      "chunk total 10000000\n",
      "chunk total 11000000\n",
      "chunk total 12000000\n",
      "chunk total 13000000\n",
      "chunk total 14000000\n",
      "chunk total 15000000\n",
      "chunk total 16000000\n",
      "chunk total 17000000\n",
      "chunk total 18000000\n",
      "chunk total 19000000\n",
      "chunk total 20000000\n",
      "chunk total 21000000\n",
      "chunk total 22000000\n",
      "chunk total 23000000\n",
      "chunk total 24000000\n",
      "chunk total 25000000\n",
      "chunk total 26000000\n",
      "chunk total 27000000\n",
      "chunk total 28000000\n",
      "chunk total 29000000\n",
      "chunk total 30000000\n",
      "chunk total 31000000\n",
      "chunk total 32000000\n",
      "chunk total 33000000\n",
      "chunk total 34000000\n",
      "chunk total 35000000\n",
      "chunk total 36000000\n",
      "chunk total 37000000\n",
      "chunk total 38000000\n",
      "chunk total 39000000\n",
      "chunk total 40000000\n",
      "chunk total 41000000\n",
      "chunk total 42000000\n",
      "chunk total 43000000\n",
      "chunk total 44000000\n",
      "chunk total 45000000\n",
      "chunk total 46000000\n",
      "chunk total 47000000\n",
      "chunk total 48000000\n",
      "chunk total 49000000\n",
      "chunk total 50000000\n",
      "chunk total 51000000\n",
      "chunk total 52000000\n",
      "chunk total 53000000\n",
      "chunk total 54000000\n",
      "chunk total 55000000\n",
      "chunk total 56000000\n",
      "chunk total 57000000\n",
      "chunk total 58000000\n",
      "chunk total 59000000\n",
      "chunk total 60000000\n",
      "chunk total 61000000\n",
      "chunk total 62000000\n",
      "chunk total 63000000\n",
      "chunk total 64000000\n",
      "chunk total 65000000\n",
      "chunk total 66000000\n",
      "chunk total 67000000\n",
      "chunk total 68000000\n",
      "chunk total 69000000\n",
      "chunk total 70000000\n",
      "chunk total 71000000\n",
      "chunk total 72000000\n",
      "chunk total 73000000\n",
      "chunk total 74000000\n",
      "chunk total 75000000\n",
      "chunk total 76000000\n",
      "chunk total 77000000\n",
      "chunk total 78000000\n",
      "chunk total 79000000\n",
      "chunk total 80000000\n",
      "chunk total 81000000\n",
      "chunk total 82000000\n",
      "chunk total 83000000\n",
      "chunk total 84000000\n",
      "chunk total 85000000\n",
      "chunk total 86000000\n",
      "chunk total 87000000\n",
      "chunk total 88000000\n",
      "chunk total 89000000\n",
      "chunk total 90000000\n",
      "chunk total 91000000\n",
      "chunk total 92000000\n",
      "chunk total 93000000\n",
      "chunk total 94000000\n",
      "chunk total 95000000\n",
      "chunk total 96000000\n",
      "chunk total 97000000\n",
      "chunk total 98000000\n",
      "chunk total 99000000\n",
      "chunk total 100000000\n",
      "chunk total 101000000\n",
      "chunk total 102000000\n",
      "chunk total 103000000\n",
      "chunk total 104000000\n",
      "chunk total 105000000\n",
      "chunk total 106000000\n",
      "chunk total 107000000\n",
      "chunk total 108000000\n",
      "chunk total 109000000\n",
      "chunk total 110000000\n",
      "chunk total 111000000\n",
      "chunk total 112000000\n",
      "chunk total 113000000\n",
      "chunk total 114000000\n",
      "chunk total 115000000\n",
      "chunk total 116000000\n",
      "chunk total 117000000\n",
      "chunk total 118000000\n",
      "chunk total 119000000\n",
      "chunk total 120000000\n",
      "chunk total 121000000\n",
      "chunk total 122000000\n",
      "chunk total 123000000\n",
      "chunk total 124000000\n",
      "chunk total 125000000\n",
      "chunk total 126000000\n",
      "chunk total 127000000\n",
      "chunk total 128000000\n",
      "chunk total 129000000\n",
      "chunk total 130000000\n",
      "chunk total 131000000\n",
      "chunk total 132000000\n",
      "chunk total 133000000\n",
      "chunk total 134000000\n",
      "chunk total 135000000\n",
      "chunk total 136000000\n",
      "chunk total 137000000\n",
      "chunk total 138000000\n",
      "chunk total 139000000\n",
      "chunk total 140000000\n",
      "chunk total 141000000\n",
      "chunk total 142000000\n",
      "chunk total 143000000\n",
      "chunk total 144000000\n",
      "chunk total 145000000\n",
      "chunk total 146000000\n",
      "chunk total 147000000\n",
      "chunk total 148000000\n",
      "chunk total 149000000\n",
      "chunk total 150000000\n",
      "chunk total 151000000\n",
      "chunk total 152000000\n",
      "chunk total 153000000\n",
      "chunk total 154000000\n",
      "chunk total 155000000\n",
      "chunk total 156000000\n",
      "chunk total 157000000\n",
      "chunk total 158000000\n",
      "chunk total 159000000\n",
      "chunk total 160000000\n",
      "chunk total 161000000\n",
      "chunk total 162000000\n",
      "chunk total 163000000\n",
      "chunk total 164000000\n",
      "chunk total 165000000\n",
      "chunk total 166000000\n",
      "chunk total 167000000\n",
      "chunk total 168000000\n",
      "chunk total 169000000\n",
      "chunk total 170000000\n",
      "chunk total 171000000\n",
      "chunk total 172000000\n",
      "chunk total 173000000\n",
      "chunk total 174000000\n",
      "chunk total 175000000\n",
      "chunk total 176000000\n",
      "chunk total 177000000\n",
      "chunk total 178000000\n",
      "chunk total 179000000\n",
      "chunk total 180000000\n",
      "chunk total 181000000\n",
      "chunk total 182000000\n",
      "chunk total 183000000\n",
      "chunk total 184000000\n",
      "chunk total 185000000\n",
      "chunk total 186000000\n",
      "chunk total 187000000\n",
      "chunk total 188000000\n",
      "chunk total 189000000\n",
      "chunk total 190000000\n",
      "chunk total 191000000\n",
      "chunk total 192000000\n",
      "chunk total 193000000\n",
      "chunk total 194000000\n",
      "chunk total 195000000\n",
      "chunk total 196000000\n",
      "chunk total 197000000\n",
      "chunk total 198000000\n",
      "chunk total 199000000\n",
      "chunk total 200000000\n",
      "chunk total 201000000\n",
      "chunk total 202000000\n",
      "chunk total 203000000\n",
      "chunk total 204000000\n",
      "chunk total 205000000\n",
      "chunk total 206000000\n",
      "chunk total 207000000\n",
      "chunk total 208000000\n",
      "chunk total 209000000\n",
      "chunk total 210000000\n",
      "chunk total 211000000\n",
      "chunk total 212000000\n",
      "chunk total 213000000\n",
      "chunk total 214000000\n",
      "chunk total 215000000\n",
      "chunk total 216000000\n",
      "chunk total 217000000\n",
      "chunk total 218000000\n",
      "chunk total 219000000\n",
      "chunk total 220000000\n",
      "chunk total 221000000\n",
      "chunk total 222000000\n",
      "chunk total 223000000\n",
      "chunk total 224000000\n",
      "chunk total 225000000\n",
      "chunk total 226000000\n",
      "chunk total 227000000\n",
      "chunk total 228000000\n",
      "chunk total 229000000\n",
      "chunk total 230000000\n",
      "chunk total 231000000\n",
      "chunk total 232000000\n",
      "chunk total 233000000\n",
      "chunk total 234000000\n",
      "chunk total 235000000\n",
      "chunk total 236000000\n",
      "chunk total 237000000\n",
      "chunk total 238000000\n",
      "chunk total 239000000\n",
      "chunk total 240000000\n",
      "chunk total 241000000\n",
      "chunk total 242000000\n",
      "chunk total 243000000\n",
      "chunk total 244000000\n",
      "chunk total 245000000\n",
      "chunk total 246000000\n",
      "chunk total 247000000\n",
      "chunk total 248000000\n",
      "chunk total 249000000\n",
      "chunk total 250000000\n",
      "chunk total 251000000\n",
      "chunk total 252000000\n",
      "chunk total 253000000\n",
      "chunk total 254000000\n",
      "chunk total 255000000\n",
      "chunk total 256000000\n",
      "chunk total 257000000\n",
      "chunk total 258000000\n",
      "chunk total 259000000\n",
      "chunk total 260000000\n",
      "chunk total 261000000\n",
      "chunk total 262000000\n",
      "chunk total 263000000\n",
      "chunk total 264000000\n",
      "chunk total 265000000\n",
      "chunk total 266000000\n",
      "chunk total 267000000\n",
      "chunk total 268000000\n",
      "chunk total 269000000\n",
      "chunk total 270000000\n",
      "chunk total 271000000\n",
      "chunk total 272000000\n",
      "chunk total 273000000\n",
      "chunk total 274000000\n",
      "chunk total 275000000\n",
      "chunk total 276000000\n",
      "chunk total 277000000\n",
      "chunk total 278000000\n",
      "chunk total 279000000\n",
      "chunk total 280000000\n",
      "chunk total 281000000\n",
      "chunk total 282000000\n",
      "chunk total 283000000\n",
      "chunk total 284000000\n",
      "chunk total 285000000\n",
      "chunk total 286000000\n",
      "chunk total 287000000\n",
      "chunk total 288000000\n",
      "chunk total 289000000\n",
      "chunk total 290000000\n",
      "chunk total 291000000\n",
      "chunk total 292000000\n",
      "chunk total 293000000\n",
      "chunk total 294000000\n",
      "chunk total 295000000\n",
      "chunk total 296000000\n",
      "chunk total 297000000\n",
      "chunk total 298000000\n",
      "chunk total 299000000\n",
      "chunk total 300000000\n",
      "chunk total 301000000\n",
      "chunk total 302000000\n",
      "chunk total 303000000\n",
      "chunk total 304000000\n",
      "chunk total 305000000\n",
      "chunk total 306000000\n",
      "chunk total 307000000\n",
      "chunk total 308000000\n",
      "chunk total 309000000\n",
      "chunk total 310000000\n",
      "chunk total 311000000\n",
      "chunk total 312000000\n",
      "chunk total 313000000\n",
      "chunk total 314000000\n",
      "chunk total 315000000\n",
      "chunk total 316000000\n",
      "chunk total 317000000\n",
      "chunk total 318000000\n",
      "chunk total 319000000\n",
      "chunk total 320000000\n",
      "chunk total 321000000\n",
      "chunk total 322000000\n",
      "chunk total 323000000\n",
      "chunk total 324000000\n",
      "chunk total 325000000\n",
      "chunk total 326000000\n",
      "chunk total 327000000\n",
      "chunk total 328000000\n",
      "chunk total 329000000\n",
      "chunk total 330000000\n",
      "chunk total 331000000\n",
      "chunk total 332000000\n",
      "chunk total 333000000\n",
      "chunk total 334000000\n",
      "chunk total 335000000\n",
      "chunk total 336000000\n",
      "chunk total 337000000\n",
      "chunk total 338000000\n",
      "chunk total 339000000\n",
      "chunk total 340000000\n",
      "chunk total 341000000\n",
      "chunk total 342000000\n",
      "chunk total 343000000\n",
      "chunk total 344000000\n",
      "chunk total 345000000\n",
      "chunk total 346000000\n",
      "chunk total 347000000\n",
      "chunk total 348000000\n",
      "chunk total 349000000\n",
      "chunk total 350000000\n",
      "chunk total 351000000\n",
      "chunk total 352000000\n",
      "chunk total 353000000\n",
      "chunk total 354000000\n",
      "chunk total 355000000\n",
      "chunk total 356000000\n",
      "chunk total 357000000\n",
      "chunk total 358000000\n",
      "chunk total 359000000\n",
      "chunk total 360000000\n",
      "chunk total 361000000\n",
      "chunk total 362000000\n",
      "chunk total 363000000\n",
      "chunk total 364000000\n",
      "chunk total 365000000\n",
      "chunk total 366000000\n",
      "chunk total 367000000\n",
      "chunk total 368000000\n",
      "chunk total 369000000\n",
      "chunk total 370000000\n",
      "chunk total 371000000\n",
      "chunk total 372000000\n",
      "chunk total 373000000\n",
      "chunk total 374000000\n",
      "chunk total 375000000\n",
      "chunk total 376000000\n",
      "chunk total 377000000\n",
      "chunk total 378000000\n",
      "chunk total 379000000\n",
      "chunk total 380000000\n",
      "chunk total 381000000\n",
      "chunk total 382000000\n",
      "chunk total 383000000\n",
      "chunk total 384000000\n",
      "chunk total 385000000\n",
      "chunk total 386000000\n",
      "chunk total 387000000\n",
      "chunk total 388000000\n",
      "chunk total 389000000\n",
      "chunk total 390000000\n",
      "chunk total 391000000\n",
      "chunk total 392000000\n",
      "chunk total 393000000\n",
      "chunk total 394000000\n",
      "chunk total 395000000\n",
      "chunk total 396000000\n",
      "chunk total 397000000\n",
      "chunk total 398000000\n",
      "chunk total 399000000\n",
      "chunk total 400000000\n",
      "chunk total 401000000\n",
      "chunk total 402000000\n",
      "chunk total 403000000\n",
      "chunk total 404000000\n",
      "chunk total 405000000\n",
      "chunk total 406000000\n",
      "chunk total 407000000\n",
      "chunk total 408000000\n",
      "chunk total 409000000\n",
      "chunk total 410000000\n",
      "chunk total 411000000\n",
      "chunk total 412000000\n",
      "chunk total 413000000\n",
      "chunk total 414000000\n",
      "chunk total 415000000\n",
      "chunk total 416000000\n",
      "chunk total 417000000\n",
      "chunk total 418000000\n",
      "chunk total 419000000\n",
      "chunk total 420000000\n",
      "chunk total 421000000\n",
      "chunk total 422000000\n",
      "chunk total 423000000\n",
      "chunk total 424000000\n",
      "chunk total 425000000\n",
      "chunk total 426000000\n",
      "chunk total 427000000\n",
      "chunk total 428000000\n",
      "chunk total 429000000\n",
      "chunk total 430000000\n",
      "chunk total 431000000\n",
      "chunk total 432000000\n",
      "chunk total 433000000\n",
      "chunk total 434000000\n",
      "chunk total 435000000\n",
      "chunk total 436000000\n",
      "chunk total 437000000\n",
      "chunk total 438000000\n",
      "chunk total 439000000\n",
      "chunk total 440000000\n",
      "chunk total 441000000\n",
      "chunk total 442000000\n",
      "chunk total 443000000\n",
      "chunk total 444000000\n",
      "chunk total 445000000\n",
      "chunk total 446000000\n",
      "chunk total 447000000\n",
      "chunk total 448000000\n",
      "chunk total 449000000\n",
      "chunk total 450000000\n",
      "chunk total 451000000\n",
      "chunk total 452000000\n",
      "chunk total 453000000\n",
      "chunk total 454000000\n",
      "chunk total 455000000\n",
      "chunk total 456000000\n",
      "chunk total 457000000\n",
      "chunk total 458000000\n",
      "chunk total 459000000\n",
      "chunk total 460000000\n",
      "chunk total 461000000\n",
      "chunk total 462000000\n",
      "chunk total 463000000\n",
      "chunk total 464000000\n",
      "chunk total 465000000\n",
      "chunk total 466000000\n",
      "chunk total 467000000\n",
      "chunk total 468000000\n",
      "chunk total 469000000\n",
      "chunk total 470000000\n",
      "chunk total 471000000\n",
      "chunk total 472000000\n",
      "chunk total 473000000\n",
      "chunk total 474000000\n",
      "chunk total 475000000\n",
      "chunk total 476000000\n",
      "chunk total 477000000\n",
      "chunk total 478000000\n",
      "chunk total 479000000\n",
      "chunk total 480000000\n",
      "chunk total 481000000\n",
      "chunk total 482000000\n",
      "chunk total 483000000\n",
      "chunk total 484000000\n",
      "chunk total 485000000\n",
      "chunk total 486000000\n",
      "chunk total 487000000\n",
      "chunk total 488000000\n",
      "chunk total 489000000\n",
      "chunk total 490000000\n",
      "chunk total 491000000\n",
      "chunk total 492000000\n",
      "chunk total 493000000\n",
      "chunk total 494000000\n",
      "chunk total 495000000\n",
      "chunk total 496000000\n",
      "chunk total 497000000\n",
      "chunk total 498000000\n",
      "chunk total 499000000\n",
      "chunk total 500000000\n",
      "chunk total 501000000\n",
      "chunk total 502000000\n",
      "chunk total 503000000\n",
      "chunk total 504000000\n",
      "chunk total 505000000\n",
      "chunk total 506000000\n",
      "chunk total 507000000\n",
      "chunk total 508000000\n",
      "chunk total 509000000\n",
      "chunk total 510000000\n",
      "chunk total 511000000\n",
      "chunk total 512000000\n",
      "chunk total 513000000\n",
      "chunk total 514000000\n",
      "chunk total 515000000\n",
      "chunk total 516000000\n",
      "chunk total 517000000\n",
      "chunk total 518000000\n",
      "chunk total 519000000\n",
      "chunk total 520000000\n",
      "chunk total 521000000\n",
      "chunk total 522000000\n",
      "chunk total 523000000\n",
      "chunk total 524000000\n",
      "chunk total 525000000\n",
      "chunk total 526000000\n",
      "chunk total 527000000\n",
      "chunk total 528000000\n",
      "chunk total 529000000\n",
      "chunk total 530000000\n",
      "chunk total 531000000\n",
      "chunk total 532000000\n",
      "chunk total 533000000\n",
      "chunk total 534000000\n",
      "chunk total 535000000\n",
      "chunk total 536000000\n",
      "chunk total 537000000\n",
      "chunk total 538000000\n",
      "chunk total 539000000\n",
      "chunk total 540000000\n",
      "chunk total 541000000\n",
      "chunk total 542000000\n",
      "chunk total 543000000\n",
      "chunk total 544000000\n",
      "chunk total 545000000\n",
      "chunk total 546000000\n",
      "chunk total 547000000\n",
      "chunk total 548000000\n",
      "chunk total 549000000\n",
      "chunk total 550000000\n",
      "chunk total 551000000\n",
      "chunk total 552000000\n",
      "chunk total 553000000\n",
      "chunk total 554000000\n",
      "chunk total 555000000\n",
      "chunk total 556000000\n",
      "chunk total 557000000\n",
      "chunk total 558000000\n",
      "chunk total 559000000\n",
      "chunk total 560000000\n",
      "chunk total 561000000\n",
      "chunk total 562000000\n",
      "chunk total 563000000\n",
      "chunk total 564000000\n",
      "chunk total 565000000\n",
      "chunk total 566000000\n",
      "chunk total 567000000\n",
      "chunk total 568000000\n",
      "chunk total 569000000\n",
      "chunk total 570000000\n",
      "chunk total 571000000\n",
      "chunk total 572000000\n",
      "chunk total 573000000\n",
      "chunk total 574000000\n",
      "chunk total 575000000\n",
      "chunk total 576000000\n",
      "chunk total 577000000\n",
      "chunk total 578000000\n",
      "chunk total 579000000\n",
      "chunk total 580000000\n",
      "chunk total 581000000\n",
      "chunk total 582000000\n",
      "chunk total 583000000\n",
      "chunk total 584000000\n",
      "chunk total 585000000\n",
      "chunk total 586000000\n",
      "chunk total 587000000\n",
      "chunk total 588000000\n",
      "chunk total 589000000\n",
      "chunk total 590000000\n",
      "chunk total 591000000\n",
      "chunk total 592000000\n",
      "chunk total 593000000\n",
      "chunk total 594000000\n",
      "chunk total 595000000\n",
      "chunk total 596000000\n",
      "chunk total 597000000\n",
      "chunk total 598000000\n",
      "chunk total 599000000\n",
      "chunk total 600000000\n",
      "chunk total 601000000\n",
      "chunk total 602000000\n",
      "chunk total 603000000\n",
      "chunk total 604000000\n",
      "chunk total 605000000\n",
      "chunk total 606000000\n",
      "chunk total 607000000\n",
      "chunk total 608000000\n",
      "chunk total 609000000\n",
      "chunk total 610000000\n",
      "chunk total 611000000\n",
      "chunk total 612000000\n",
      "chunk total 613000000\n",
      "chunk total 614000000\n",
      "chunk total 615000000\n",
      "chunk total 616000000\n",
      "chunk total 617000000\n",
      "chunk total 618000000\n",
      "chunk total 619000000\n",
      "chunk total 620000000\n",
      "chunk total 621000000\n",
      "chunk total 622000000\n",
      "chunk total 623000000\n",
      "chunk total 624000000\n",
      "chunk total 625000000\n",
      "chunk total 626000000\n",
      "chunk total 627000000\n",
      "chunk total 628000000\n",
      "chunk total 629000000\n",
      "chunk total 630000000\n",
      "chunk total 631000000\n",
      "chunk total 632000000\n",
      "chunk total 633000000\n",
      "chunk total 634000000\n",
      "chunk total 635000000\n",
      "chunk total 636000000\n",
      "chunk total 637000000\n",
      "chunk total 638000000\n",
      "chunk total 639000000\n",
      "chunk total 640000000\n",
      "chunk total 641000000\n",
      "chunk total 642000000\n",
      "chunk total 643000000\n",
      "chunk total 644000000\n",
      "chunk total 645000000\n",
      "chunk total 646000000\n",
      "chunk total 647000000\n",
      "chunk total 648000000\n",
      "chunk total 649000000\n",
      "chunk total 650000000\n",
      "chunk total 651000000\n",
      "chunk total 652000000\n",
      "chunk total 653000000\n",
      "chunk total 654000000\n",
      "chunk total 655000000\n",
      "chunk total 656000000\n",
      "chunk total 657000000\n",
      "chunk total 658000000\n",
      "chunk total 659000000\n",
      "chunk total 660000000\n",
      "chunk total 661000000\n",
      "chunk total 662000000\n",
      "chunk total 663000000\n",
      "chunk total 664000000\n",
      "chunk total 665000000\n",
      "chunk total 666000000\n",
      "chunk total 667000000\n",
      "chunk total 668000000\n",
      "chunk total 669000000\n",
      "chunk total 670000000\n",
      "chunk total 671000000\n",
      "chunk total 672000000\n",
      "chunk total 673000000\n",
      "chunk total 674000000\n",
      "chunk total 675000000\n",
      "chunk total 676000000\n",
      "chunk total 677000000\n",
      "chunk total 678000000\n",
      "chunk total 679000000\n",
      "chunk total 680000000\n",
      "chunk total 681000000\n",
      "chunk total 682000000\n",
      "chunk total 683000000\n",
      "chunk total 684000000\n",
      "chunk total 685000000\n",
      "chunk total 686000000\n",
      "chunk total 687000000\n",
      "chunk total 688000000\n",
      "chunk total 689000000\n",
      "chunk total 690000000\n",
      "chunk total 691000000\n",
      "chunk total 692000000\n",
      "chunk total 693000000\n",
      "chunk total 694000000\n",
      "chunk total 695000000\n",
      "chunk total 696000000\n",
      "chunk total 697000000\n",
      "chunk total 698000000\n",
      "chunk total 699000000\n",
      "chunk total 700000000\n",
      "chunk total 701000000\n",
      "chunk total 702000000\n",
      "chunk total 703000000\n",
      "chunk total 704000000\n",
      "chunk total 705000000\n",
      "chunk total 706000000\n",
      "chunk total 707000000\n",
      "chunk total 708000000\n",
      "chunk total 709000000\n",
      "chunk total 710000000\n",
      "chunk total 711000000\n",
      "chunk total 712000000\n",
      "chunk total 713000000\n",
      "chunk total 714000000\n",
      "chunk total 715000000\n",
      "chunk total 716000000\n",
      "chunk total 717000000\n",
      "chunk total 718000000\n",
      "chunk total 719000000\n",
      "chunk total 720000000\n",
      "chunk total 721000000\n",
      "chunk total 722000000\n",
      "chunk total 723000000\n",
      "chunk total 724000000\n",
      "chunk total 725000000\n",
      "chunk total 726000000\n",
      "chunk total 727000000\n",
      "chunk total 728000000\n",
      "chunk total 729000000\n",
      "chunk total 730000000\n",
      "chunk total 731000000\n",
      "chunk total 732000000\n",
      "chunk total 733000000\n",
      "chunk total 734000000\n",
      "chunk total 735000000\n",
      "chunk total 736000000\n",
      "chunk total 737000000\n",
      "chunk total 738000000\n",
      "chunk total 739000000\n",
      "chunk total 740000000\n",
      "chunk total 741000000\n",
      "chunk total 742000000\n",
      "chunk total 743000000\n",
      "chunk total 744000000\n",
      "chunk total 745000000\n",
      "chunk total 746000000\n",
      "chunk total 747000000\n",
      "chunk total 748000000\n",
      "chunk total 749000000\n",
      "chunk total 750000000\n",
      "chunk total 751000000\n",
      "chunk total 752000000\n",
      "chunk total 753000000\n",
      "chunk total 754000000\n",
      "chunk total 755000000\n",
      "chunk total 756000000\n",
      "chunk total 757000000\n",
      "chunk total 758000000\n",
      "chunk total 759000000\n",
      "chunk total 760000000\n",
      "chunk total 761000000\n",
      "chunk total 762000000\n",
      "chunk total 763000000\n",
      "chunk total 764000000\n",
      "chunk total 765000000\n",
      "chunk total 766000000\n",
      "chunk total 767000000\n",
      "chunk total 768000000\n",
      "chunk total 769000000\n",
      "chunk total 770000000\n",
      "chunk total 771000000\n",
      "chunk total 772000000\n",
      "chunk total 773000000\n",
      "chunk total 774000000\n",
      "chunk total 775000000\n",
      "chunk total 776000000\n",
      "chunk total 777000000\n",
      "chunk total 778000000\n",
      "chunk total 779000000\n",
      "chunk total 780000000\n",
      "chunk total 781000000\n",
      "chunk total 782000000\n",
      "chunk total 783000000\n",
      "chunk total 784000000\n",
      "chunk total 785000000\n",
      "chunk total 786000000\n",
      "chunk total 787000000\n",
      "chunk total 788000000\n",
      "chunk total 789000000\n",
      "chunk total 790000000\n",
      "chunk total 791000000\n",
      "chunk total 792000000\n",
      "chunk total 793000000\n",
      "chunk total 794000000\n",
      "chunk total 795000000\n",
      "chunk total 796000000\n",
      "chunk total 797000000\n",
      "chunk total 798000000\n",
      "chunk total 799000000\n",
      "chunk total 800000000\n",
      "chunk total 801000000\n",
      "chunk total 802000000\n",
      "chunk total 803000000\n",
      "chunk total 804000000\n",
      "chunk total 805000000\n",
      "chunk total 806000000\n",
      "chunk total 807000000\n",
      "chunk total 808000000\n",
      "chunk total 809000000\n",
      "chunk total 810000000\n",
      "chunk total 811000000\n",
      "chunk total 812000000\n",
      "chunk total 813000000\n",
      "chunk total 814000000\n",
      "chunk total 815000000\n",
      "chunk total 816000000\n",
      "chunk total 817000000\n",
      "chunk total 818000000\n",
      "chunk total 819000000\n",
      "chunk total 820000000\n",
      "chunk total 821000000\n",
      "chunk total 822000000\n",
      "chunk total 823000000\n",
      "chunk total 824000000\n",
      "chunk total 825000000\n",
      "chunk total 826000000\n",
      "chunk total 827000000\n",
      "chunk total 828000000\n",
      "chunk total 829000000\n",
      "chunk total 830000000\n",
      "chunk total 831000000\n",
      "chunk total 832000000\n",
      "chunk total 833000000\n",
      "chunk total 834000000\n",
      "chunk total 835000000\n",
      "chunk total 836000000\n",
      "chunk total 837000000\n",
      "chunk total 838000000\n",
      "chunk total 839000000\n",
      "chunk total 840000000\n",
      "chunk total 841000000\n",
      "chunk total 842000000\n",
      "chunk total 843000000\n",
      "chunk total 844000000\n",
      "chunk total 845000000\n",
      "chunk total 846000000\n",
      "chunk total 847000000\n",
      "chunk total 848000000\n",
      "chunk total 849000000\n",
      "chunk total 850000000\n",
      "chunk total 851000000\n",
      "chunk total 852000000\n",
      "chunk total 853000000\n",
      "chunk total 854000000\n",
      "chunk total 855000000\n",
      "chunk total 856000000\n",
      "chunk total 857000000\n",
      "chunk total 858000000\n",
      "chunk total 859000000\n",
      "chunk total 860000000\n",
      "chunk total 861000000\n",
      "chunk total 862000000\n",
      "chunk total 863000000\n",
      "chunk total 864000000\n",
      "chunk total 865000000\n",
      "chunk total 866000000\n",
      "chunk total 867000000\n",
      "chunk total 868000000\n",
      "chunk total 869000000\n",
      "chunk total 870000000\n",
      "chunk total 871000000\n",
      "chunk total 872000000\n",
      "chunk total 873000000\n",
      "chunk total 874000000\n",
      "chunk total 875000000\n",
      "chunk total 876000000\n",
      "chunk total 877000000\n",
      "chunk total 878000000\n",
      "chunk total 879000000\n",
      "chunk total 880000000\n",
      "chunk total 881000000\n",
      "chunk total 882000000\n",
      "chunk total 883000000\n",
      "chunk total 884000000\n",
      "chunk total 885000000\n",
      "chunk total 886000000\n",
      "chunk total 887000000\n",
      "chunk total 888000000\n",
      "chunk total 889000000\n",
      "chunk total 890000000\n",
      "chunk total 891000000\n",
      "chunk total 892000000\n",
      "chunk total 893000000\n",
      "chunk total 894000000\n",
      "chunk total 895000000\n",
      "chunk total 896000000\n",
      "chunk total 897000000\n",
      "chunk total 898000000\n",
      "chunk total 899000000\n",
      "chunk total 900000000\n",
      "chunk total 901000000\n",
      "chunk total 902000000\n",
      "chunk total 903000000\n",
      "chunk total 904000000\n",
      "chunk total 905000000\n",
      "chunk total 906000000\n",
      "chunk total 907000000\n",
      "chunk total 908000000\n",
      "chunk total 909000000\n",
      "chunk total 910000000\n",
      "chunk total 911000000\n",
      "chunk total 912000000\n",
      "chunk total 913000000\n",
      "chunk total 914000000\n",
      "chunk total 915000000\n",
      "chunk total 916000000\n",
      "chunk total 917000000\n",
      "chunk total 918000000\n",
      "chunk total 919000000\n",
      "chunk total 920000000\n",
      "chunk total 921000000\n",
      "chunk total 922000000\n",
      "chunk total 923000000\n",
      "chunk total 924000000\n",
      "chunk total 925000000\n",
      "chunk total 926000000\n",
      "chunk total 927000000\n",
      "chunk total 928000000\n",
      "chunk total 929000000\n",
      "chunk total 930000000\n",
      "chunk total 931000000\n",
      "chunk total 932000000\n",
      "chunk total 933000000\n",
      "chunk total 934000000\n",
      "chunk total 935000000\n",
      "chunk total 936000000\n",
      "chunk total 937000000\n",
      "chunk total 938000000\n",
      "chunk total 939000000\n",
      "chunk total 940000000\n",
      "chunk total 941000000\n",
      "chunk total 942000000\n",
      "chunk total 943000000\n",
      "chunk total 944000000\n",
      "chunk total 945000000\n",
      "chunk total 946000000\n",
      "chunk total 947000000\n",
      "chunk total 948000000\n",
      "chunk total 949000000\n",
      "chunk total 950000000\n",
      "chunk total 951000000\n",
      "chunk total 952000000\n",
      "chunk total 953000000\n",
      "chunk total 954000000\n",
      "chunk total 955000000\n",
      "chunk total 956000000\n",
      "chunk total 957000000\n",
      "chunk total 958000000\n",
      "chunk total 959000000\n",
      "chunk total 960000000\n",
      "chunk total 961000000\n",
      "chunk total 962000000\n",
      "chunk total 963000000\n",
      "chunk total 964000000\n",
      "chunk total 965000000\n",
      "chunk total 966000000\n",
      "chunk total 967000000\n",
      "chunk total 968000000\n",
      "chunk total 969000000\n",
      "chunk total 970000000\n",
      "chunk total 971000000\n",
      "chunk total 972000000\n",
      "chunk total 973000000\n",
      "chunk total 974000000\n",
      "chunk total 975000000\n",
      "chunk total 976000000\n",
      "chunk total 977000000\n",
      "chunk total 978000000\n",
      "chunk total 979000000\n",
      "chunk total 980000000\n",
      "chunk total 981000000\n",
      "chunk total 982000000\n",
      "chunk total 983000000\n",
      "chunk total 984000000\n",
      "chunk total 985000000\n",
      "chunk total 986000000\n",
      "chunk total 987000000\n",
      "chunk total 988000000\n",
      "chunk total 989000000\n",
      "chunk total 990000000\n",
      "chunk total 991000000\n",
      "chunk total 992000000\n",
      "chunk total 993000000\n",
      "chunk total 994000000\n",
      "chunk total 995000000\n",
      "chunk total 996000000\n",
      "chunk total 997000000\n",
      "chunk total 998000000\n",
      "chunk total 999000000\n",
      "chunk total 1000000000\n",
      "chunk total 1001000000\n",
      "chunk total 1002000000\n",
      "chunk total 1003000000\n",
      "chunk total 1004000000\n",
      "chunk total 1005000000\n",
      "chunk total 1006000000\n",
      "chunk total 1007000000\n",
      "chunk total 1008000000\n",
      "chunk total 1009000000\n",
      "chunk total 1010000000\n",
      "chunk total 1011000000\n",
      "chunk total 1012000000\n",
      "chunk total 1013000000\n",
      "chunk total 1014000000\n",
      "chunk total 1015000000\n",
      "chunk total 1016000000\n",
      "chunk total 1017000000\n",
      "chunk total 1018000000\n",
      "chunk total 1019000000\n",
      "chunk total 1020000000\n",
      "chunk total 1021000000\n",
      "chunk total 1022000000\n",
      "chunk total 1023000000\n",
      "chunk total 1024000000\n",
      "chunk total 1025000000\n",
      "chunk total 1026000000\n",
      "chunk total 1027000000\n",
      "chunk total 1028000000\n",
      "chunk total 1029000000\n",
      "chunk total 1030000000\n",
      "chunk total 1031000000\n",
      "chunk total 1032000000\n",
      "chunk total 1033000000\n",
      "chunk total 1034000000\n",
      "chunk total 1035000000\n",
      "chunk total 1036000000\n",
      "chunk total 1037000000\n",
      "chunk total 1038000000\n",
      "chunk total 1039000000\n",
      "chunk total 1040000000\n",
      "chunk total 1041000000\n",
      "chunk total 1042000000\n",
      "chunk total 1043000000\n",
      "chunk total 1044000000\n",
      "chunk total 1045000000\n",
      "chunk total 1046000000\n",
      "chunk total 1047000000\n",
      "chunk total 1048000000\n",
      "chunk total 1049000000\n",
      "chunk total 1050000000\n",
      "chunk total 1051000000\n",
      "chunk total 1052000000\n",
      "chunk total 1053000000\n",
      "chunk total 1054000000\n",
      "chunk total 1055000000\n",
      "chunk total 1056000000\n",
      "chunk total 1057000000\n",
      "chunk total 1058000000\n",
      "chunk total 1059000000\n",
      "chunk total 1060000000\n",
      "chunk total 1061000000\n",
      "chunk total 1062000000\n",
      "chunk total 1063000000\n",
      "chunk total 1064000000\n",
      "chunk total 1065000000\n",
      "chunk total 1066000000\n",
      "chunk total 1067000000\n",
      "chunk total 1068000000\n",
      "chunk total 1069000000\n",
      "chunk total 1070000000\n",
      "chunk total 1071000000\n",
      "chunk total 1072000000\n",
      "chunk total 1073000000\n",
      "chunk total 1074000000\n",
      "chunk total 1075000000\n",
      "chunk total 1076000000\n",
      "chunk total 1077000000\n",
      "chunk total 1078000000\n",
      "chunk total 1079000000\n",
      "chunk total 1080000000\n",
      "chunk total 1081000000\n",
      "chunk total 1082000000\n",
      "chunk total 1083000000\n",
      "chunk total 1084000000\n",
      "chunk total 1085000000\n",
      "chunk total 1086000000\n",
      "chunk total 1087000000\n",
      "chunk total 1088000000\n",
      "chunk total 1089000000\n",
      "chunk total 1090000000\n",
      "chunk total 1091000000\n",
      "chunk total 1092000000\n",
      "chunk total 1093000000\n",
      "chunk total 1094000000\n",
      "chunk total 1095000000\n",
      "chunk total 1096000000\n",
      "chunk total 1097000000\n",
      "chunk total 1098000000\n",
      "chunk total 1099000000\n",
      "chunk total 1100000000\n",
      "chunk total 1101000000\n",
      "chunk total 1102000000\n",
      "chunk total 1103000000\n",
      "chunk total 1104000000\n",
      "chunk total 1105000000\n",
      "chunk total 1106000000\n",
      "chunk total 1107000000\n",
      "chunk total 1108000000\n",
      "chunk total 1109000000\n",
      "chunk total 1110000000\n",
      "chunk total 1111000000\n",
      "chunk total 1112000000\n",
      "chunk total 1113000000\n",
      "chunk total 1114000000\n",
      "chunk total 1115000000\n",
      "chunk total 1116000000\n",
      "chunk total 1117000000\n",
      "chunk total 1118000000\n",
      "chunk total 1119000000\n",
      "chunk total 1120000000\n",
      "chunk total 1121000000\n",
      "chunk total 1122000000\n",
      "chunk total 1123000000\n",
      "chunk total 1124000000\n",
      "chunk total 1125000000\n",
      "chunk total 1126000000\n",
      "chunk total 1127000000\n",
      "chunk total 1128000000\n",
      "chunk total 1129000000\n",
      "chunk total 1130000000\n",
      "chunk total 1131000000\n",
      "chunk total 1132000000\n",
      "chunk total 1133000000\n",
      "chunk total 1134000000\n",
      "chunk total 1135000000\n",
      "chunk total 1136000000\n",
      "chunk total 1137000000\n",
      "chunk total 1138000000\n",
      "chunk total 1139000000\n",
      "chunk total 1140000000\n",
      "chunk total 1141000000\n",
      "chunk total 1142000000\n",
      "chunk total 1143000000\n",
      "chunk total 1144000000\n",
      "chunk total 1145000000\n",
      "chunk total 1146000000\n",
      "chunk total 1147000000\n",
      "chunk total 1148000000\n",
      "chunk total 1149000000\n",
      "chunk total 1150000000\n",
      "chunk total 1151000000\n",
      "chunk total 1152000000\n",
      "chunk total 1153000000\n",
      "chunk total 1154000000\n",
      "chunk total 1155000000\n",
      "chunk total 1156000000\n",
      "chunk total 1157000000\n",
      "chunk total 1158000000\n",
      "chunk total 1159000000\n",
      "chunk total 1160000000\n",
      "chunk total 1161000000\n",
      "chunk total 1162000000\n",
      "chunk total 1163000000\n",
      "chunk total 1164000000\n",
      "chunk total 1165000000\n",
      "chunk total 1166000000\n",
      "chunk total 1167000000\n",
      "chunk total 1168000000\n",
      "chunk total 1169000000\n",
      "chunk total 1170000000\n",
      "chunk total 1171000000\n",
      "chunk total 1172000000\n",
      "chunk total 1173000000\n",
      "chunk total 1174000000\n",
      "chunk total 1175000000\n",
      "chunk total 1176000000\n",
      "chunk total 1177000000\n",
      "chunk total 1178000000\n",
      "chunk total 1179000000\n",
      "chunk total 1180000000\n",
      "chunk total 1181000000\n",
      "chunk total 1182000000\n",
      "chunk total 1183000000\n",
      "chunk total 1184000000\n",
      "chunk total 1185000000\n",
      "chunk total 1186000000\n",
      "chunk total 1187000000\n",
      "chunk total 1188000000\n",
      "chunk total 1189000000\n",
      "chunk total 1190000000\n",
      "chunk total 1191000000\n",
      "chunk total 1192000000\n",
      "chunk total 1193000000\n",
      "chunk total 1194000000\n",
      "chunk total 1195000000\n",
      "chunk total 1196000000\n",
      "chunk total 1197000000\n",
      "chunk total 1198000000\n",
      "chunk total 1199000000\n",
      "chunk total 1200000000\n",
      "chunk total 1201000000\n",
      "chunk total 1202000000\n",
      "chunk total 1203000000\n",
      "chunk total 1204000000\n",
      "chunk total 1205000000\n",
      "chunk total 1206000000\n",
      "chunk total 1207000000\n",
      "chunk total 1208000000\n",
      "chunk total 1209000000\n",
      "chunk total 1210000000\n",
      "chunk total 1211000000\n",
      "chunk total 1212000000\n",
      "chunk total 1213000000\n",
      "chunk total 1214000000\n",
      "chunk total 1215000000\n",
      "chunk total 1216000000\n",
      "chunk total 1217000000\n",
      "chunk total 1218000000\n",
      "chunk total 1219000000\n",
      "chunk total 1220000000\n",
      "chunk total 1221000000\n",
      "chunk total 1222000000\n",
      "chunk total 1223000000\n",
      "chunk total 1224000000\n",
      "chunk total 1225000000\n",
      "chunk total 1226000000\n",
      "chunk total 1227000000\n",
      "chunk total 1228000000\n",
      "chunk total 1229000000\n",
      "chunk total 1230000000\n",
      "chunk total 1231000000\n",
      "chunk total 1232000000\n",
      "chunk total 1233000000\n",
      "chunk total 1234000000\n",
      "chunk total 1235000000\n",
      "chunk total 1236000000\n",
      "chunk total 1237000000\n",
      "chunk total 1238000000\n",
      "chunk total 1239000000\n",
      "chunk total 1240000000\n",
      "chunk total 1241000000\n",
      "chunk total 1242000000\n",
      "chunk total 1243000000\n",
      "chunk total 1244000000\n",
      "chunk total 1245000000\n",
      "chunk total 1246000000\n",
      "chunk total 1247000000\n",
      "chunk total 1248000000\n",
      "chunk total 1249000000\n",
      "chunk total 1250000000\n",
      "chunk total 1251000000\n",
      "chunk total 1252000000\n",
      "chunk total 1253000000\n",
      "chunk total 1254000000\n",
      "chunk total 1255000000\n",
      "chunk total 1256000000\n",
      "chunk total 1257000000\n",
      "chunk total 1258000000\n",
      "chunk total 1259000000\n",
      "chunk total 1260000000\n",
      "chunk total 1261000000\n",
      "chunk total 1262000000\n",
      "chunk total 1263000000\n",
      "chunk total 1264000000\n",
      "chunk total 1265000000\n",
      "chunk total 1266000000\n",
      "chunk total 1267000000\n",
      "chunk total 1268000000\n",
      "chunk total 1269000000\n",
      "chunk total 1270000000\n",
      "chunk total 1271000000\n",
      "chunk total 1272000000\n",
      "chunk total 1273000000\n",
      "chunk total 1274000000\n",
      "chunk total 1275000000\n",
      "chunk total 1276000000\n",
      "chunk total 1277000000\n",
      "chunk total 1278000000\n",
      "chunk total 1279000000\n",
      "chunk total 1280000000\n",
      "chunk total 1281000000\n",
      "chunk total 1282000000\n",
      "chunk total 1283000000\n",
      "chunk total 1284000000\n",
      "chunk total 1285000000\n",
      "chunk total 1286000000\n",
      "chunk total 1287000000\n",
      "chunk total 1288000000\n",
      "chunk total 1289000000\n",
      "chunk total 1290000000\n",
      "chunk total 1291000000\n",
      "chunk total 1292000000\n",
      "chunk total 1293000000\n",
      "chunk total 1294000000\n",
      "chunk total 1295000000\n",
      "chunk total 1296000000\n",
      "chunk total 1297000000\n",
      "chunk total 1298000000\n",
      "chunk total 1299000000\n",
      "chunk total 1300000000\n",
      "chunk total 1301000000\n",
      "chunk total 1302000000\n",
      "chunk total 1303000000\n",
      "chunk total 1304000000\n",
      "chunk total 1305000000\n",
      "chunk total 1306000000\n",
      "chunk total 1307000000\n",
      "chunk total 1308000000\n",
      "chunk total 1309000000\n",
      "chunk total 1310000000\n",
      "chunk total 1311000000\n",
      "chunk total 1312000000\n",
      "chunk total 1313000000\n",
      "chunk total 1314000000\n",
      "chunk total 1315000000\n",
      "chunk total 1316000000\n",
      "chunk total 1317000000\n",
      "chunk total 1318000000\n",
      "chunk total 1319000000\n",
      "chunk total 1320000000\n",
      "chunk total 1321000000\n",
      "chunk total 1322000000\n",
      "chunk total 1323000000\n",
      "chunk total 1324000000\n",
      "chunk total 1325000000\n",
      "chunk total 1326000000\n",
      "chunk total 1327000000\n",
      "chunk total 1328000000\n",
      "chunk total 1329000000\n",
      "chunk total 1330000000\n",
      "chunk total 1331000000\n",
      "chunk total 1332000000\n",
      "chunk total 1333000000\n",
      "chunk total 1334000000\n",
      "chunk total 1335000000\n",
      "chunk total 1336000000\n",
      "chunk total 1337000000\n",
      "chunk total 1338000000\n",
      "chunk total 1339000000\n",
      "chunk total 1340000000\n",
      "chunk total 1341000000\n",
      "chunk total 1342000000\n",
      "chunk total 1343000000\n",
      "chunk total 1344000000\n",
      "chunk total 1345000000\n",
      "chunk total 1346000000\n",
      "chunk total 1347000000\n",
      "chunk total 1348000000\n",
      "chunk total 1349000000\n",
      "chunk total 1350000000\n",
      "chunk total 1351000000\n",
      "chunk total 1352000000\n",
      "chunk total 1353000000\n",
      "chunk total 1354000000\n",
      "chunk total 1355000000\n",
      "chunk total 1356000000\n",
      "chunk total 1357000000\n",
      "chunk total 1358000000\n",
      "chunk total 1359000000\n",
      "chunk total 1360000000\n",
      "chunk total 1361000000\n",
      "chunk total 1362000000\n",
      "chunk total 1363000000\n",
      "chunk total 1364000000\n",
      "chunk total 1365000000\n",
      "chunk total 1366000000\n",
      "chunk total 1367000000\n",
      "chunk total 1368000000\n",
      "chunk total 1369000000\n",
      "chunk total 1370000000\n",
      "chunk total 1371000000\n",
      "chunk total 1372000000\n",
      "chunk total 1373000000\n",
      "chunk total 1374000000\n",
      "chunk total 1375000000\n",
      "chunk total 1376000000\n",
      "chunk total 1377000000\n",
      "chunk total 1378000000\n",
      "chunk total 1379000000\n",
      "chunk total 1380000000\n",
      "chunk total 1381000000\n",
      "chunk total 1382000000\n",
      "chunk total 1383000000\n",
      "chunk total 1384000000\n",
      "chunk total 1385000000\n",
      "chunk total 1386000000\n",
      "chunk total 1387000000\n",
      "chunk total 1388000000\n",
      "chunk total 1389000000\n",
      "chunk total 1390000000\n",
      "chunk total 1391000000\n",
      "chunk total 1392000000\n",
      "chunk total 1393000000\n",
      "chunk total 1394000000\n",
      "chunk total 1395000000\n",
      "chunk total 1396000000\n",
      "chunk total 1397000000\n",
      "chunk total 1398000000\n",
      "chunk total 1399000000\n",
      "chunk total 1400000000\n",
      "chunk total 1401000000\n",
      "chunk total 1402000000\n",
      "chunk total 1403000000\n",
      "chunk total 1404000000\n",
      "chunk total 1405000000\n",
      "chunk total 1406000000\n",
      "chunk total 1407000000\n",
      "chunk total 1408000000\n",
      "chunk total 1409000000\n",
      "chunk total 1410000000\n",
      "chunk total 1411000000\n",
      "chunk total 1412000000\n",
      "chunk total 1413000000\n",
      "chunk total 1414000000\n",
      "chunk total 1415000000\n",
      "chunk total 1416000000\n",
      "chunk total 1417000000\n",
      "chunk total 1418000000\n",
      "chunk total 1419000000\n",
      "chunk total 1420000000\n",
      "chunk total 1421000000\n",
      "chunk total 1422000000\n",
      "chunk total 1423000000\n",
      "chunk total 1424000000\n",
      "chunk total 1425000000\n",
      "chunk total 1426000000\n",
      "chunk total 1427000000\n",
      "chunk total 1428000000\n",
      "chunk total 1429000000\n",
      "chunk total 1430000000\n",
      "chunk total 1431000000\n",
      "chunk total 1432000000\n",
      "chunk total 1433000000\n",
      "chunk total 1434000000\n",
      "chunk total 1435000000\n",
      "chunk total 1436000000\n",
      "chunk total 1437000000\n",
      "chunk total 1438000000\n",
      "chunk total 1439000000\n",
      "chunk total 1440000000\n",
      "chunk total 1441000000\n",
      "chunk total 1442000000\n",
      "chunk total 1443000000\n",
      "chunk total 1444000000\n",
      "chunk total 1445000000\n",
      "chunk total 1446000000\n",
      "chunk total 1447000000\n",
      "chunk total 1448000000\n",
      "chunk total 1449000000\n",
      "chunk total 1450000000\n",
      "chunk total 1451000000\n",
      "chunk total 1452000000\n",
      "chunk total 1453000000\n",
      "chunk total 1454000000\n",
      "chunk total 1455000000\n",
      "chunk total 1456000000\n",
      "chunk total 1457000000\n",
      "chunk total 1458000000\n",
      "chunk total 1459000000\n",
      "chunk total 1460000000\n",
      "chunk total 1461000000\n",
      "chunk total 1462000000\n",
      "chunk total 1463000000\n",
      "chunk total 1464000000\n",
      "chunk total 1465000000\n",
      "chunk total 1466000000\n",
      "chunk total 1467000000\n",
      "chunk total 1468000000\n",
      "chunk total 1469000000\n",
      "chunk total 1470000000\n",
      "chunk total 1471000000\n",
      "chunk total 1472000000\n",
      "chunk total 1473000000\n",
      "chunk total 1474000000\n",
      "chunk total 1475000000\n",
      "chunk total 1476000000\n",
      "chunk total 1477000000\n",
      "chunk total 1478000000\n",
      "chunk total 1479000000\n",
      "chunk total 1480000000\n",
      "chunk total 1481000000\n",
      "chunk total 1482000000\n",
      "chunk total 1483000000\n",
      "chunk total 1484000000\n",
      "chunk total 1485000000\n",
      "chunk total 1486000000\n",
      "chunk total 1487000000\n",
      "chunk total 1488000000\n",
      "chunk total 1489000000\n",
      "chunk total 1490000000\n",
      "chunk total 1491000000\n",
      "chunk total 1492000000\n",
      "chunk total 1493000000\n",
      "chunk total 1494000000\n",
      "chunk total 1495000000\n",
      "chunk total 1496000000\n",
      "chunk total 1497000000\n",
      "chunk total 1498000000\n",
      "chunk total 1499000000\n",
      "chunk total 1500000000\n",
      "chunk total 1501000000\n",
      "chunk total 1502000000\n",
      "chunk total 1503000000\n",
      "chunk total 1504000000\n",
      "chunk total 1505000000\n",
      "chunk total 1506000000\n",
      "chunk total 1507000000\n",
      "chunk total 1508000000\n",
      "chunk total 1509000000\n",
      "chunk total 1510000000\n",
      "chunk total 1511000000\n",
      "chunk total 1512000000\n",
      "chunk total 1513000000\n",
      "chunk total 1514000000\n",
      "chunk total 1515000000\n",
      "chunk total 1516000000\n",
      "chunk total 1517000000\n",
      "chunk total 1518000000\n",
      "chunk total 1519000000\n",
      "chunk total 1520000000\n",
      "chunk total 1521000000\n",
      "chunk total 1522000000\n",
      "chunk total 1523000000\n",
      "chunk total 1524000000\n",
      "chunk total 1525000000\n",
      "chunk total 1526000000\n",
      "chunk total 1527000000\n",
      "chunk total 1528000000\n",
      "chunk total 1529000000\n",
      "chunk total 1530000000\n",
      "chunk total 1531000000\n",
      "chunk total 1532000000\n",
      "chunk total 1533000000\n",
      "chunk total 1534000000\n",
      "chunk total 1535000000\n",
      "chunk total 1536000000\n",
      "chunk total 1537000000\n",
      "chunk total 1538000000\n",
      "chunk total 1539000000\n",
      "chunk total 1540000000\n",
      "chunk total 1541000000\n",
      "chunk total 1542000000\n",
      "chunk total 1543000000\n",
      "chunk total 1544000000\n",
      "chunk total 1545000000\n",
      "chunk total 1546000000\n",
      "chunk total 1547000000\n",
      "chunk total 1547741381\n"
     ]
    }
   ],
   "source": [
    "import duckdb\n",
    "import pyarrow.dataset as ds\n",
    "\n",
    "nyc = ds.dataset(\"nyc-taxi/\", partitioning=[\"year\", \"month\"])\n",
    "\n",
    "con = duckdb.connect()\n",
    "query = con.execute(\"SELECT total_amount FROM nyc\")\n",
    "\n",
    "# Create Record Batch Reader from Query Result.\n",
    "record_batch_reader = query.fetch_record_batch()\n",
    "\n",
    "total_rows = 0\n",
    "while True:\n",
    "    print('chunk total', total_rows)\n",
    "    try:\n",
    "        # Process a single chunk here\n",
    "        # pyarrow.lib.RecordBatch\n",
    "        chunk = record_batch_reader.read_next_batch()\n",
    "        total_rows += len(chunk)\n",
    "    except StopIteration:\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "eRohGKD7Kc_I"
   },
   "source": [
    "## NYC Taxi with DuckDB and polars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "9qHYsgZOPCbh"
   },
   "outputs": [],
   "source": [
    "import duckdb\n",
    "import pyarrow as pa\n",
    "import pyarrow.dataset as ds\n",
    "import polars as pl\n",
    "from time import perf_counter as timer\n",
    "\n",
    "def analyze_nyc_taxi(min_year=2014, min_amount=100):\n",
    "    \"\"\"\n",
    "    Analyze NYC taxi data with specified filters using Polars LazyFrame\n",
    "    \"\"\"\n",
    "    print(\"Starting analysis...\")\n",
    "    start_time = timer()\n",
    "\n",
    "    # Open dataset using year,month folder partition\n",
    "    nyc = ds.dataset(\"nyc-taxi/\", partitioning=[\"year\", \"month\"])\n",
    "\n",
    "    # Get database connection\n",
    "    con = duckdb.connect()\n",
    "    query= f\"\"\"\n",
    "        SELECT\n",
    "            total_amount,\n",
    "            passenger_count,\n",
    "            year,\n",
    "            pickup_at,\n",
    "            pickup_longitude as lon,\n",
    "            pickup_latitude as lat\n",
    "        FROM nyc\n",
    "        WHERE total_amount > {min_amount}\n",
    "            AND year >= {min_year}\n",
    "            AND lat is not null\n",
    "            AND lon is not null\n",
    "    \"\"\"\n",
    "    req = con.execute(query)\n",
    "\n",
    "    # Create Record Batch Reader and convert to Polars\n",
    "    record_batch_reader = req.fetch_record_batch()\n",
    "\n",
    "    # Process chunks and convert to Polars LazyFrame\n",
    "    all_chunks = []\n",
    "    total_rows = 0\n",
    "\n",
    "    while True:\n",
    "        try:\n",
    "            chunk = record_batch_reader.read_next_batch()\n",
    "            # Convert PyArrow RecordBatch to Polars DataFrame\n",
    "            pl_chunk = pl.from_arrow(pa.Table.from_batches([chunk]))\n",
    "            all_chunks.append(pl_chunk)\n",
    "            total_rows += len(pl_chunk)\n",
    "            print(f\"Processed chunk, total rows: {total_rows}\")\n",
    "        except StopIteration:\n",
    "            break\n",
    "\n",
    "    end_time = timer()\n",
    "    runtime = end_time - start_time\n",
    "\n",
    "    # Combine all chunks into a single Polars DataFrame\n",
    "    lf = pl.concat(all_chunks).lazy()\n",
    "\n",
    "    print(f\"\\nAnalysis completed in {runtime:.2f} seconds\")\n",
    "    print(f\"Total rows processed: {total_rows}\")\n",
    "\n",
    "    return lf, runtime\n",
    "\n",
    "def get_summary_stats(lf):\n",
    "    \"\"\"Calculate statistics total_amount using LazyFrame operations\"\"\"\n",
    "    stats = (\n",
    "        lf.select([\n",
    "            pl.col(\"total_amount\").mean().alias(\"avg_fare\"),\n",
    "            pl.col(\"total_amount\").min().alias(\"min_fare\"),\n",
    "            pl.col(\"total_amount\").max().alias(\"max_fare\"),\n",
    "            pl.col(\"passenger_count\").mean().alias(\"avg_passengers\"),\n",
    "            pl.col(\"year\").value_counts().alias(\"trips_by_year\")\n",
    "        ])\n",
    "    ).collect()\n",
    "\n",
    "    return stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000,
     "referenced_widgets": [
      "2f3193b5fc1243b19740d5e9afbbc57f",
      "5877791d426e45e99d8d8f56bb5ee027",
      "36f0aec83403481cbac93304c4566979"
     ]
    },
    "id": "dUATeKLuQmp3",
    "outputId": "c01856c6-05b1-4177-c8a9-141bda45f382"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting analysis...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2f3193b5fc1243b19740d5e9afbbc57f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed chunk, total rows: 1000000\n",
      "Processed chunk, total rows: 2000000\n",
      "Processed chunk, total rows: 3000000\n",
      "Processed chunk, total rows: 4000000\n",
      "Processed chunk, total rows: 5000000\n",
      "Processed chunk, total rows: 6000000\n",
      "Processed chunk, total rows: 7000000\n",
      "Processed chunk, total rows: 8000000\n",
      "Processed chunk, total rows: 9000000\n",
      "Processed chunk, total rows: 10000000\n",
      "Processed chunk, total rows: 11000000\n",
      "Processed chunk, total rows: 12000000\n",
      "Processed chunk, total rows: 13000000\n",
      "Processed chunk, total rows: 14000000\n",
      "Processed chunk, total rows: 15000000\n",
      "Processed chunk, total rows: 16000000\n",
      "Processed chunk, total rows: 17000000\n",
      "Processed chunk, total rows: 18000000\n",
      "Processed chunk, total rows: 19000000\n",
      "Processed chunk, total rows: 20000000\n",
      "Processed chunk, total rows: 21000000\n",
      "Processed chunk, total rows: 22000000\n",
      "Processed chunk, total rows: 23000000\n",
      "Processed chunk, total rows: 24000000\n",
      "Processed chunk, total rows: 25000000\n",
      "Processed chunk, total rows: 26000000\n",
      "Processed chunk, total rows: 27000000\n",
      "Processed chunk, total rows: 28000000\n",
      "Processed chunk, total rows: 29000000\n",
      "Processed chunk, total rows: 30000000\n",
      "Processed chunk, total rows: 31000000\n",
      "Processed chunk, total rows: 32000000\n",
      "Processed chunk, total rows: 33000000\n",
      "Processed chunk, total rows: 34000000\n",
      "Processed chunk, total rows: 35000000\n",
      "Processed chunk, total rows: 36000000\n",
      "Processed chunk, total rows: 37000000\n",
      "Processed chunk, total rows: 38000000\n",
      "Processed chunk, total rows: 39000000\n",
      "Processed chunk, total rows: 40000000\n",
      "Processed chunk, total rows: 41000000\n",
      "Processed chunk, total rows: 42000000\n",
      "Processed chunk, total rows: 43000000\n",
      "Processed chunk, total rows: 44000000\n",
      "Processed chunk, total rows: 45000000\n",
      "Processed chunk, total rows: 46000000\n",
      "Processed chunk, total rows: 47000000\n",
      "Processed chunk, total rows: 48000000\n",
      "Processed chunk, total rows: 49000000\n",
      "Processed chunk, total rows: 50000000\n",
      "Processed chunk, total rows: 51000000\n",
      "Processed chunk, total rows: 52000000\n",
      "Processed chunk, total rows: 53000000\n",
      "Processed chunk, total rows: 54000000\n",
      "Processed chunk, total rows: 55000000\n",
      "Processed chunk, total rows: 56000000\n",
      "Processed chunk, total rows: 57000000\n",
      "Processed chunk, total rows: 58000000\n",
      "Processed chunk, total rows: 59000000\n",
      "Processed chunk, total rows: 60000000\n",
      "Processed chunk, total rows: 61000000\n",
      "Processed chunk, total rows: 62000000\n",
      "Processed chunk, total rows: 63000000\n",
      "Processed chunk, total rows: 63477143\n",
      "\n",
      "Analysis completed in 231.54 seconds\n",
      "Total rows processed: 63477143\n",
      "CPU times: user 2min 20s, sys: 37.8 s, total: 2min 58s\n",
      "Wall time: 3min 51s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "lf, runtime = analyze_nyc_taxi(min_year=2009, min_amount=35) #3.5 min with min_year=2009, min_amount=35"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 168
    },
    "id": "kMiAL006RICz",
    "outputId": "04f7e3f0-f481-415a-a4ce-b892c6ffc83b"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<h4>NAIVE QUERY PLAN</h4><p>run <b>LazyFrame.show_graph()</b> to see the optimized version</p><?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.43.0 (0)\n",
       " -->\n",
       "<!-- Title: polars_query Pages: 1 -->\n",
       "<svg width=\"67pt\" height=\"61pt\"\n",
       " viewBox=\"0.00 0.00 67.00 61.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 57)\">\n",
       "<title>polars_query</title>\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-57 63,-57 63,4 -4,4\"/>\n",
       "<!-- p1 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>p1</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"59,-53 0,-53 0,0 59,0 59,-53\"/>\n",
       "<text text-anchor=\"middle\" x=\"29.5\" y=\"-37.8\" font-family=\"Times,serif\" font-size=\"14.00\">TABLE</text>\n",
       "<text text-anchor=\"middle\" x=\"29.5\" y=\"-22.8\" font-family=\"Times,serif\" font-size=\"14.00\">π */6;</text>\n",
       "<text text-anchor=\"middle\" x=\"29.5\" y=\"-7.8\" font-family=\"Times,serif\" font-size=\"14.00\">σ None</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<LazyFrame at 0x787D4A51F880>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 443
    },
    "id": "n_o3QcLiS_Og",
    "outputId": "2ac8343f-4de2-4740-f012-f628e5a4b35b"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (63_477_143, 6)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>total_amount</th><th>passenger_count</th><th>year</th><th>pickup_at</th><th>lon</th><th>lat</th></tr><tr><td>f32</td><td>i8</td><td>i32</td><td>datetime[μs]</td><td>f32</td><td>f32</td></tr></thead><tbody><tr><td>35.450001</td><td>1</td><td>2009</td><td>2009-01-21 14:38:05</td><td>-73.997314</td><td>40.736099</td></tr><tr><td>38.220001</td><td>1</td><td>2009</td><td>2009-01-06 14:51:53</td><td>-73.862762</td><td>40.769268</td></tr><tr><td>45.0</td><td>2</td><td>2009</td><td>2009-01-03 16:27:10</td><td>-73.984085</td><td>40.756809</td></tr><tr><td>45.0</td><td>1</td><td>2009</td><td>2009-01-03 16:48:23</td><td>-74.006645</td><td>40.716652</td></tr><tr><td>51.75</td><td>1</td><td>2009</td><td>2009-01-07 15:22:59</td><td>-73.998528</td><td>40.723438</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>58.34</td><td>1</td><td>2016</td><td>2016-06-21 22:08:36</td><td>-73.784401</td><td>40.648598</td></tr><tr><td>36.84</td><td>1</td><td>2016</td><td>2016-06-21 21:50:55</td><td>-73.863342</td><td>40.769947</td></tr><tr><td>36.34</td><td>1</td><td>2016</td><td>2016-06-21 21:50:55</td><td>-73.871033</td><td>40.773754</td></tr><tr><td>73.300003</td><td>1</td><td>2016</td><td>2016-06-21 22:01:39</td><td>-73.783401</td><td>40.64875</td></tr><tr><td>80.800003</td><td>1</td><td>2016</td><td>2016-06-21 22:01:39</td><td>-73.792809</td><td>40.658192</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (63_477_143, 6)\n",
       "┌──────────────┬─────────────────┬──────┬─────────────────────┬────────────┬───────────┐\n",
       "│ total_amount ┆ passenger_count ┆ year ┆ pickup_at           ┆ lon        ┆ lat       │\n",
       "│ ---          ┆ ---             ┆ ---  ┆ ---                 ┆ ---        ┆ ---       │\n",
       "│ f32          ┆ i8              ┆ i32  ┆ datetime[μs]        ┆ f32        ┆ f32       │\n",
       "╞══════════════╪═════════════════╪══════╪═════════════════════╪════════════╪═══════════╡\n",
       "│ 35.450001    ┆ 1               ┆ 2009 ┆ 2009-01-21 14:38:05 ┆ -73.997314 ┆ 40.736099 │\n",
       "│ 38.220001    ┆ 1               ┆ 2009 ┆ 2009-01-06 14:51:53 ┆ -73.862762 ┆ 40.769268 │\n",
       "│ 45.0         ┆ 2               ┆ 2009 ┆ 2009-01-03 16:27:10 ┆ -73.984085 ┆ 40.756809 │\n",
       "│ 45.0         ┆ 1               ┆ 2009 ┆ 2009-01-03 16:48:23 ┆ -74.006645 ┆ 40.716652 │\n",
       "│ 51.75        ┆ 1               ┆ 2009 ┆ 2009-01-07 15:22:59 ┆ -73.998528 ┆ 40.723438 │\n",
       "│ …            ┆ …               ┆ …    ┆ …                   ┆ …          ┆ …         │\n",
       "│ 58.34        ┆ 1               ┆ 2016 ┆ 2016-06-21 22:08:36 ┆ -73.784401 ┆ 40.648598 │\n",
       "│ 36.84        ┆ 1               ┆ 2016 ┆ 2016-06-21 21:50:55 ┆ -73.863342 ┆ 40.769947 │\n",
       "│ 36.34        ┆ 1               ┆ 2016 ┆ 2016-06-21 21:50:55 ┆ -73.871033 ┆ 40.773754 │\n",
       "│ 73.300003    ┆ 1               ┆ 2016 ┆ 2016-06-21 22:01:39 ┆ -73.783401 ┆ 40.64875  │\n",
       "│ 80.800003    ┆ 1               ┆ 2016 ┆ 2016-06-21 22:01:39 ┆ -73.792809 ┆ 40.658192 │\n",
       "└──────────────┴─────────────────┴──────┴─────────────────────┴────────────┴───────────┘"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Convert LazyFrame to regular Polars DataFrame\n",
    "df = lf.collect()\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "M4TLpu_uQt92",
    "outputId": "6986b388-37b8-4d65-aa68-de60f790880f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Summary Statistics:\n",
      "shape: (8, 5)\n",
      "┌───────────┬───────────┬───────────┬────────────────┬─────────────────┐\n",
      "│ avg_fare  ┆ min_fare  ┆ max_fare  ┆ avg_passengers ┆ trips_by_year   │\n",
      "│ ---       ┆ ---       ┆ ---       ┆ ---            ┆ ---             │\n",
      "│ f32       ┆ f32       ┆ f32       ┆ f64            ┆ struct[2]       │\n",
      "╞═══════════╪═══════════╪═══════════╪════════════════╪═════════════════╡\n",
      "│ 51.979137 ┆ 35.009998 ┆ 3950611.5 ┆ 1.727499       ┆ {2009,4980036}  │\n",
      "│ 51.979137 ┆ 35.009998 ┆ 3950611.5 ┆ 1.727499       ┆ {2012,8158817}  │\n",
      "│ 51.979137 ┆ 35.009998 ┆ 3950611.5 ┆ 1.727499       ┆ {2013,10895158} │\n",
      "│ 51.979137 ┆ 35.009998 ┆ 3950611.5 ┆ 1.727499       ┆ {2016,5458275}  │\n",
      "│ 51.979137 ┆ 35.009998 ┆ 3950611.5 ┆ 1.727499       ┆ {2014,11332771} │\n",
      "│ 51.979137 ┆ 35.009998 ┆ 3950611.5 ┆ 1.727499       ┆ {2011,6534332}  │\n",
      "│ 51.979137 ┆ 35.009998 ┆ 3950611.5 ┆ 1.727499       ┆ {2010,4908528}  │\n",
      "│ 51.979137 ┆ 35.009998 ┆ 3950611.5 ┆ 1.727499       ┆ {2015,11209226} │\n",
      "└───────────┴───────────┴───────────┴────────────────┴─────────────────┘\n"
     ]
    }
   ],
   "source": [
    "stats = get_summary_stats(lf)\n",
    "print(\"Summary Statistics:\")\n",
    "print(stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "XkD0Tvj1Qt6r",
    "outputId": "b54bf1b0-eb0a-4791-8299-853a135fb6ba"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Yearly Statistics:\n",
      "shape: (8, 3)\n",
      "┌──────┬───────────┬────────────┐\n",
      "│ year ┆ avg_fare  ┆ trip_count │\n",
      "│ ---  ┆ ---       ┆ ---        │\n",
      "│ i32  ┆ f64       ┆ u32        │\n",
      "╞══════╪═══════════╪════════════╡\n",
      "│ 2009 ┆ 48.593525 ┆ 4980036    │\n",
      "│ 2010 ┆ 48.843003 ┆ 4908528    │\n",
      "│ 2011 ┆ 49.288672 ┆ 6534332    │\n",
      "│ 2012 ┆ 49.862339 ┆ 8158817    │\n",
      "│ 2013 ┆ 51.888556 ┆ 10895158   │\n",
      "│ 2014 ┆ 52.842987 ┆ 11332771   │\n",
      "│ 2015 ┆ 54.677651 ┆ 11209226   │\n",
      "│ 2016 ┆ 54.179889 ┆ 5458275    │\n",
      "└──────┴───────────┴────────────┘\n"
     ]
    }
   ],
   "source": [
    "# Get average fare by year\n",
    "yearly_stats = (\n",
    "    lf.group_by(\"year\")\n",
    "    .agg([\n",
    "        pl.col(\"total_amount\").mean().alias(\"avg_fare\"),\n",
    "        pl.len().alias(\"trip_count\")\n",
    "    ])\n",
    "    .sort(\"year\")\n",
    "    .collect()\n",
    ")\n",
    "print(\"\\nYearly Statistics:\")\n",
    "print(yearly_stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "sjxheZk7Qt2I"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "R1ZX4TcVYuDe"
   },
   "source": [
    "## with Polars secco (2X LENTO RISPETTO DuckDB)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "AvERlkkjX8sC"
   },
   "outputs": [],
   "source": [
    "import polars as pl\n",
    "import pyarrow.dataset as ds\n",
    "from time import perf_counter as timer\n",
    "from pathlib import Path\n",
    "\n",
    "def create_lazy_dataset(data_path: str) -> pl.LazyFrame:\n",
    "    \"\"\"\n",
    "    Create a LazyFrame from the partitioned dataset\n",
    "    \"\"\"\n",
    "    # Create dataset with partitioning\n",
    "    nyc = ds.dataset(data_path, partitioning=[\"year\", \"month\"])\n",
    "\n",
    "    # Convert to Polars LazyFrame\n",
    "    return pl.scan_pyarrow_dataset(nyc)\n",
    "\n",
    "def build_analysis_query(lf: pl.LazyFrame, min_year: int = 2014, min_amount: float = 100) -> pl.LazyFrame:\n",
    "    \"\"\"\n",
    "    Build the main analysis query keeping everything lazy\n",
    "    \"\"\"\n",
    "    return (\n",
    "        lf.select([\n",
    "            \"total_amount\",\n",
    "            \"passenger_count\",\n",
    "            \"year\",\n",
    "            \"month\",\n",
    "            \"pickup_at\",\n",
    "            \"pickup_longitude\",\n",
    "            \"pickup_latitude\"\n",
    "        ])\n",
    "        .filter(\n",
    "            (pl.col(\"total_amount\") > min_amount) &\n",
    "            (pl.col(\"year\") >= min_year) &\n",
    "            (pl.col(\"pickup_latitude\").is_not_null()) &\n",
    "            (pl.col(\"pickup_longitude\").is_not_null())\n",
    "        )\n",
    "        .with_columns([\n",
    "            pl.col(\"pickup_longitude\").alias(\"lon\"),\n",
    "            pl.col(\"pickup_latitude\").alias(\"lat\")\n",
    "        ])\n",
    "    )\n",
    "\n",
    "def get_basic_stats(lf: pl.LazyFrame) -> pl.LazyFrame:\n",
    "    \"\"\"\n",
    "    Create a LazyFrame for basic statistics\n",
    "    \"\"\"\n",
    "    return (\n",
    "        lf.select([\n",
    "            pl.len().alias(\"total_rows\"),\n",
    "            pl.col(\"total_amount\").mean().alias(\"avg_fare\"),\n",
    "            pl.col(\"total_amount\").min().alias(\"min_fare\"),\n",
    "            pl.col(\"total_amount\").max().alias(\"max_fare\"),\n",
    "            pl.col(\"passenger_count\").mean().alias(\"avg_passengers\"),\n",
    "        ])\n",
    "    )\n",
    "\n",
    "def get_yearly_stats(lf: pl.LazyFrame) -> pl.LazyFrame:\n",
    "    \"\"\"\n",
    "    Create a LazyFrame for yearly statistics\n",
    "    \"\"\"\n",
    "    return (\n",
    "        lf.group_by(\"year\")\n",
    "        .agg([\n",
    "            pl.col(\"total_amount\").mean().alias(\"avg_fare\"),\n",
    "            pl.len().alias(\"trip_count\"),\n",
    "            pl.col(\"total_amount\").sum().alias(\"total_revenue\")\n",
    "        ])\n",
    "        .sort(\"year\")\n",
    "    )\n",
    "\n",
    "def get_monthly_patterns(lf: pl.LazyFrame) -> pl.LazyFrame:\n",
    "    \"\"\"\n",
    "    Create a LazyFrame for monthly patterns\n",
    "    \"\"\"\n",
    "    return (\n",
    "        lf.group_by([\"year\", \"month\"])\n",
    "        .agg([\n",
    "            pl.len().alias(\"trip_count\"),\n",
    "            pl.col(\"total_amount\").mean().alias(\"avg_fare\")\n",
    "        ])\n",
    "        .sort([\"year\", \"month\"])\n",
    "    )\n",
    "\n",
    "def analyze_taxi_data(data_path: str, min_year: int = 2014, min_amount: float = 100):\n",
    "    \"\"\"\n",
    "    Main analysis function that keeps operations lazy until final collection\n",
    "    \"\"\"\n",
    "    print(\"Starting analysis...\")\n",
    "    start_time = timer()\n",
    "\n",
    "    # Create base LazyFrame\n",
    "    base_lf = create_lazy_dataset(data_path)\n",
    "\n",
    "    # Build main analysis query\n",
    "    analysis_lf = build_analysis_query(base_lf, min_year, min_amount)\n",
    "\n",
    "    # Prepare all analysis queries (still lazy)\n",
    "    basic_stats_lf = get_basic_stats(analysis_lf)\n",
    "    yearly_stats_lf = get_yearly_stats(analysis_lf)\n",
    "    monthly_patterns_lf = get_monthly_patterns(analysis_lf)\n",
    "\n",
    "    # Collect results only when needed\n",
    "    results = {\n",
    "                'basic_stats': basic_stats_lf.collect(),\n",
    "                'yearly_stats': yearly_stats_lf.collect(),\n",
    "                'monthly_patterns': monthly_patterns_lf.collect(),\n",
    "                }\n",
    "    end_time = timer()\n",
    "    runtime = end_time - start_time\n",
    "\n",
    "    print(f\"\\nAnalysis completed in {runtime:.2f} seconds\")\n",
    "    print(f\"Total rows processed: {results['basic_stats']['total_rows'][0]}\")\n",
    "\n",
    "    return analysis_lf, results, runtime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "0OSVdkvKYJAn",
    "outputId": "80a2ba41-6955-4ffe-ab3f-504c678386c3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting analysis...\n",
      "\n",
      "Analysis completed in 530.88 seconds\n",
      "Total rows processed: 63477143\n"
     ]
    }
   ],
   "source": [
    "# Run the analysis\n",
    "%%time\n",
    "data_path = \"nyc-taxi/\"\n",
    "lf, results, runtime = analyze_taxi_data(data_path=data_path,min_year=2009,min_amount=35) # circa 9 min con min_year=2009,min_amount=35"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "GCqlS0rTYQFc",
    "outputId": "2c791c26-469c-430a-9697-c57c2ffa3594"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Basic Statistics:\n",
      "shape: (1, 5)\n",
      "┌────────────┬───────────┬───────────┬───────────┬────────────────┐\n",
      "│ total_rows ┆ avg_fare  ┆ min_fare  ┆ max_fare  ┆ avg_passengers │\n",
      "│ ---        ┆ ---       ┆ ---       ┆ ---       ┆ ---            │\n",
      "│ u32        ┆ f32       ┆ f32       ┆ f32       ┆ f64            │\n",
      "╞════════════╪═══════════╪═══════════╪═══════════╪════════════════╡\n",
      "│ 63477143   ┆ 51.979137 ┆ 35.009998 ┆ 3950611.5 ┆ 1.727499       │\n",
      "└────────────┴───────────┴───────────┴───────────┴────────────────┘\n"
     ]
    }
   ],
   "source": [
    "# Print basic statistics\n",
    "print(\"\\nBasic Statistics:\")\n",
    "print(results['basic_stats'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "IlJqN8WfYRPb",
    "outputId": "13bbebe8-a234-4f2f-9039-77d8e65cdb4b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Yearly Statistics:\n",
      "shape: (8, 4)\n",
      "┌──────┬───────────┬────────────┬───────────────┐\n",
      "│ year ┆ avg_fare  ┆ trip_count ┆ total_revenue │\n",
      "│ ---  ┆ ---       ┆ ---        ┆ ---           │\n",
      "│ i32  ┆ f64       ┆ u32        ┆ f32           │\n",
      "╞══════╪═══════════╪════════════╪═══════════════╡\n",
      "│ 2009 ┆ 48.593525 ┆ 4980036    ┆ 2.41997504e8  │\n",
      "│ 2010 ┆ 48.843003 ┆ 4908528    ┆ 2.39747248e8  │\n",
      "│ 2011 ┆ 49.288672 ┆ 6534332    ┆ 3.22068544e8  │\n",
      "│ 2012 ┆ 49.862339 ┆ 8158817    ┆ 4.06817696e8  │\n",
      "│ 2013 ┆ 51.888556 ┆ 10895158   ┆ 5.65334016e8  │\n",
      "│ 2014 ┆ 52.842987 ┆ 11332771   ┆ 5.98857472e8  │\n",
      "│ 2015 ┆ 54.677651 ┆ 11209226   ┆ 6.12894144e8  │\n",
      "│ 2016 ┆ 54.179889 ┆ 5458275    ┆ 2.95728736e8  │\n",
      "└──────┴───────────┴────────────┴───────────────┘\n"
     ]
    }
   ],
   "source": [
    "# Print yearly statistics\n",
    "print(\"\\nYearly Statistics:\")\n",
    "print(results['yearly_stats'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "7LhAXeYTYSle",
    "outputId": "e6ef8e22-ba9b-4b92-e032-45e007f91f94"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Monthly Patterns:\n",
      "shape: (89, 4)\n",
      "┌──────┬───────┬────────────┬───────────┐\n",
      "│ year ┆ month ┆ trip_count ┆ avg_fare  │\n",
      "│ ---  ┆ ---   ┆ ---        ┆ ---       │\n",
      "│ i32  ┆ i32   ┆ u32        ┆ f64       │\n",
      "╞══════╪═══════╪════════════╪═══════════╡\n",
      "│ 2009 ┆ 1     ┆ 324094     ┆ 48.835026 │\n",
      "│ 2009 ┆ 2     ┆ 318258     ┆ 48.758812 │\n",
      "│ 2009 ┆ 3     ┆ 394992     ┆ 48.837511 │\n",
      "│ 2009 ┆ 4     ┆ 385753     ┆ 48.78389  │\n",
      "│ 2009 ┆ 5     ┆ 425945     ┆ 48.743326 │\n",
      "│ …    ┆ …     ┆ …          ┆ …         │\n",
      "│ 2016 ┆ 2     ┆ 810478     ┆ 54.23402  │\n",
      "│ 2016 ┆ 3     ┆ 940763     ┆ 54.49375  │\n",
      "│ 2016 ┆ 4     ┆ 928302     ┆ 53.778141 │\n",
      "│ 2016 ┆ 5     ┆ 1019728    ┆ 54.236428 │\n",
      "│ 2016 ┆ 6     ┆ 954911     ┆ 56.020123 │\n",
      "└──────┴───────┴────────────┴───────────┘\n"
     ]
    }
   ],
   "source": [
    "# Print monthly patterns\n",
    "print(\"\\nMonthly Patterns:\")\n",
    "print(results['monthly_patterns'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "bC8oRV1vXEfx"
   },
   "outputs": [],
   "source": [
    "# Get average fare by year\n",
    "yearly_stats = (\n",
    "    lf.group_by(\"year\")\n",
    "    .agg([\n",
    "        pl.col(\"total_amount\").mean().alias(\"avg_fare\"),\n",
    "        pl.len().alias(\"trip_count\")\n",
    "    ])\n",
    "    .sort(\"year\")\n",
    "    .collect()\n",
    ")\n",
    "print(\"\\nYearly Statistics:\")\n",
    "print(yearly_stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "apTz6pbNQtzZ"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "name": "Un benvenuto a Colaboratory",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "15c42a076524404589e0b295e2f49198": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": "auto"
     }
    },
    "1cde922d422649f88db7a9974ccaf6e8": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_15c42a076524404589e0b295e2f49198",
      "max": 100,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_ade41a377f7c4ea2876540f9f169047c",
      "value": 100
     }
    },
    "2f3193b5fc1243b19740d5e9afbbc57f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_5877791d426e45e99d8d8f56bb5ee027",
      "max": 100,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_36f0aec83403481cbac93304c4566979",
      "value": 100
     }
    },
    "36f0aec83403481cbac93304c4566979": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": "black",
      "description_width": ""
     }
    },
    "5877791d426e45e99d8d8f56bb5ee027": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": "auto"
     }
    },
    "ade41a377f7c4ea2876540f9f169047c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": "black",
      "description_width": ""
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}