Prediction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prediction Notebook\n",
    "This notebook contains the code to produce predictions on GCM data using a pickled regressor.\n",
    "\n",
    "It computes the GCM data for each agroclimatic indicator and then predicts using those features for the chosen year. This is heavily RAM intensive, especially for a full dataset.\n",
    "\n",
    "It does no feature engineering, so it assumes the initial data has the non climatic indicators precomputed.\n",
    "\n",
    "Once the years have been predicted, for the year (e.g. 2025) it works out the mean and standard deviation for that year by using all rcps for that year and one year either side (12 in total)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas()\n",
    "import numpy as np\n",
    "import json\n",
    "import pickle\n",
    "import joblib\n",
    "import pickle\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "from shared_methods import compute_ten_day_feature, compute_seasonal_feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add the correct file information\n",
    "out_file = 'climate_monthly_seasonal_2005_2010_with_spatial_features.csv'\n",
    "\n",
    "with open('json_data/file_data.json', 'r') as outfile:\n",
    "    file_data = json.load(outfile)\n",
    "    \n",
    "seasonal_data_folder = file_data['seasonal_data_folder']\n",
    "ten_day_data_folder = file_data['ten_day_data_folder']\n",
    "crop_data_2005_file = file_data['crop_data_2005_file']\n",
    "crop_data_2010_file = file_data['crop_data_2010_file']\n",
    "spatial_attributes_folder = file_data['spatial_attributes_folder']\n",
    "ten_day_feature_files = file_data['ten_day_feature_files']\n",
    "seasonal_feature_files = file_data['seasonal_feature_files']\n",
    "spatial_attribute_files = file_data['spatial_attribute_files']\n",
    "\n",
    "out_file = f'../data/predictions.csv'\n",
    "regressor_pickle_file = '../data/regressors_0.pickle'\n",
    "data_file = '../data/deltas'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specify which columns are needed\n",
    "target = 'maize_a_2010'\n",
    "non_feature_columns = ['alloc_key', target]\n",
    "ten_day_features = ten_day_feature_files.keys()\n",
    "seasonal_features = seasonal_feature_files.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the data\n",
    "with open(regressor_pickle_file, 'rb') as handle:\n",
    "    regressor = pickle.load(handle)\n",
    "data = pd.read_csv(data_file)\n",
    "\n",
    "prediction_features = data.drop(columns=non_feature_columns).columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loop through each rcp and year, compute the agroclimatic features for each year and predict based on those\n",
    "# computed features.\n",
    "prediction_years = [2024, 2025, 2026]\n",
    "rcps = ['2p6', '4p5', '6p0', '8p5']\n",
    "model = 'hadgem'\n",
    "\n",
    "# Predict for different years\n",
    "for rcp in rcps:\n",
    "    folder = f'../data/{model}/{model}_rcp_{rcp}_agroclimatic_indicators/'\n",
    "    for year in tqdm(prediction_years):\n",
    "        print(f'Starting Model: {model} Year {year} RCP {rcp}')\n",
    "        if int(year) in range(2011,2041):\n",
    "            file_ending = '20110101-20401231_v1.nc'\n",
    "        elif int(year) in range(2041,2071):\n",
    "            file_ending = '20410101-20701231_v1.nc'\n",
    "\n",
    "        for feature in tqdm(seasonal_features):\n",
    "            # Get the correct file location for hadgem data\n",
    "            file_location = f'{folder}{feature}_C3S-glob-agric_hadgem2-es_rcp{rcp}_season_{file_ending}'\n",
    "            data = compute_seasonal_feature(feature, file_location, str(year), data, in_place_2010=True)\n",
    "\n",
    "        for feature in tqdm(ten_day_features):\n",
    "            # Get the correct file location for hadgem data\n",
    "            file_location = f'{folder}{feature}_C3S-glob-agric_hadgem2-es_rcp{rcp}_dek_{file_ending}'\n",
    "            data = compute_ten_day_feature(feature, file_location, str(year), data, in_place_2010=True)\n",
    "        data = data.interpolate(axis=1)\n",
    "\n",
    "        data[f'{rcp}_{year}_predict'] = regressor.predict(data[prediction_features])\n",
    "        print(f'Finished year {year} rcp {rcp} model {model}')\n",
    "    \n",
    "\n",
    "data.to_csv('predictions')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute the mean and standard deviation for a prediction year by using the rcps for that year and the same for\n",
    "# plus and minus one year either side of that year (12 predictions in total).\n",
    "\n",
    "prediction_years = [2025]\n",
    "for year in tqdm(prediction_years):\n",
    "    mean_included_years = [str(year), str(year + 1), str(year - 1)]\n",
    "    mean_included_year_columns = []\n",
    "    for rcp in rcps:\n",
    "        mean_included_year_columns += [f'{rcp}_{col}_predict' for col in mean_included_years]\n",
    "    data[f'{year}_mean'] = predictions[mean_included_year_columns].mean(axis=1)\n",
    "    data[f'{year}_std'] = predictions[mean_included_year_columns].std(axis=1)\n",
    "                                          \n",
    "\n",
    "data.to_csv(f'Predictions for Years: {prediction_years}.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}