1_data_download.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "826f89bb-d5eb-4a16-b4f4-9167ee4d5c50",
   "metadata": {},
   "source": [
    "## GEO-AI Challenge for Cropland Mapping by ITU"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "711f075e-9ad1-48dd-830a-775c7d07ecef",
   "metadata": {},
   "source": [
    "- This noteboook demonstrates the GEE data download and data processing\n",
    "- The data is downloaded separately for each region using only the samples provided for the competition\n",
    "- Cloud filtering using image properties were followed by manual inspection/removal of images \n",
    "\n",
    "\n",
    "\n",
    "The data preparation workflow is as follows\n",
    "- Separate provided train and test cvs by region\n",
    "- Get a bounding geometry for each region to filter bounds of Sentinel-2 search\n",
    "- Fetch Sentinel-2 Harmonized collection filtered by cloudy cover\n",
    "- Download time series array per region\n",
    "- Concatenate time series for each point into master array\n",
    "- Create spectral indices"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d300c61-5638-4a71-a652-34a943888fc6",
   "metadata": {},
   "source": [
    "## import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "340cc4c1-c8df-4858-b757-0afafa1ff429",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import argparse\n",
    "import json\n",
    "from tqdm import tqdm\n",
    "from datetime import datetime\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "import ee\n",
    "from scipy.interpolate import interp1d\n",
    "# ee.Authenticate()\n",
    "ee.Initialize(opt_url='https://earthengine-highvolume.googleapis.com')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c46b62c-6aa0-4a26-93b6-af80407d4c2e",
   "metadata": {},
   "source": [
    "## set working directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f7160d52-705c-4ff0-854a-a80b19f132f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '/app/dev/GeoITU/GeoITU_CropMapping'\n",
    "os.chdir(path)\n",
    "\n",
    "# import custom functions\n",
    "from get_geometry import get_bounds\n",
    "from utils_gee import *\n",
    "from utils_datapreparation import *"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f243a1e-9924-40b1-a049-bbbf4eeddc7d",
   "metadata": {},
   "source": [
    "## csv preparation\n",
    "\n",
    "-  split train and test csv into separate csvs for each region using lat/lon ranges\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "26beaa7d-bf4c-4e91-bdd0-107b640ac96e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "list of columns ['ID' 'Lat' 'Lon' 'Target']\n",
      "shape of train data (1500, 4)\n",
      "shape of test data (1500, 4)\n",
      "\n",
      "shape of afghan train (500, 4) test (500, 3)\n",
      "shape of iran train (500, 4) test (500, 3)\n",
      "shape of sudan train (500, 4) test (500, 3)\n"
     ]
    }
   ],
   "source": [
    "## read csv containing lat and long field\n",
    "train_data = pd.read_csv('data/Train.csv')\n",
    "test_data = pd.read_csv('data/Test.csv')\n",
    "print('list of columns', train_data.columns.values)\n",
    "print('shape of train data', train_data.shape)\n",
    "print('shape of test data', train_data.shape)\n",
    "\n",
    "\n",
    "# ----------TRAIN\n",
    "# split rows by region to allow sepearate file acquisition in earth engine\n",
    "afghan_train_csv = train_data[train_data[\"Lon\"] > 60]\n",
    "iran_train_csv =   train_data[(train_data[\"Lon\"] >40) & (train_data[\"Lon\"]<60)]\n",
    "sudan_train_csv =  train_data[train_data[\"Lat\"] < 20]\n",
    "\n",
    "\n",
    "# ----------TEST\n",
    "afghan_test_csv = test_data[test_data[\"Lon\"] > 60]\n",
    "iran_test_csv =   test_data[(test_data[\"Lon\"] >40) & (test_data[\"Lon\"]<60)]\n",
    "sudan_test_csv =  test_data[test_data[\"Lat\"] < 20]\n",
    "\n",
    "print('\\nshape of afghan train', afghan_train_csv.shape, \\\n",
    "      'test', afghan_test_csv.shape)\n",
    "print('shape of iran train', iran_train_csv.shape, \\\n",
    "      'test', iran_test_csv.shape)\n",
    "print('shape of sudan train', sudan_train_csv.shape, \\\n",
    "      'test', sudan_test_csv.shape)\n",
    "\n",
    "\n",
    "## assert that all seperations are unique in train\n",
    "assert(iran_train_csv['ID'].values != sudan_train_csv['ID'].values).all()\n",
    "assert(iran_train_csv['ID'].values != afghan_train_csv['ID'].values).all()\n",
    "\n",
    "\n",
    "## assert that all seperations are unique in test\n",
    "assert(iran_test_csv['ID'].values != sudan_test_csv['ID'].values).all()\n",
    "assert(iran_test_csv['ID'].values != afghan_test_csv['ID'].values).all()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0d0c136c-bb0a-454e-afa3-11b6406786e3",
   "metadata": {},
   "source": [
    "## save csv per region"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "dbd5e0bb-d072-4425-85a8-1bb4a7cbb67e",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Train\n",
    "# afghan_train_csv.to_csv('data/Train_afghan.csv', index=False)\n",
    "# iran_train_csv.to_csv('data/Train_iran.csv', index=False)\n",
    "# sudan_train_csv.to_csv('data/Train_sudan.csv', index=False)\n",
    "\n",
    "## Test\n",
    "# afghan_test_csv.to_csv('data/Test_afghan.csv', index=False)\n",
    "# iran_test_csv.to_csv('data/Test_iran.csv', index=False)\n",
    "# sudan_test_csv.to_csv('data/Test_sudan.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e8cdd8a-844e-4c97-a3ec-109527e0fbbb",
   "metadata": {},
   "source": [
    "## download data by region"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9e74d211-f17f-4475-9917-19dd8a21132f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get bounds for each region as ee object (pre-determined from earthengine)\n",
    "afghan_geom = get_bounds('afghan')\n",
    "iran_geom = get_bounds('iran')\n",
    "sudan_geom = get_bounds('sudan')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ddc3724b-e93e-428c-8b4e-89f361f5d37d",
   "metadata": {},
   "source": [
    "#### afghan collection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "daaf6004-10a1-41b0-9024-31e8ac5f5daf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of images for afghan region 18\n"
     ]
    }
   ],
   "source": [
    "# for Afghan region, it was observed that although label points \n",
    "# were collected in April, crops were still in season in May. \n",
    "# the search window was set from March to May\n",
    "\n",
    "# Train\n",
    "afghan_collection = prepare_collection(start_date='2022-03-01', end_date='2022-05-31', \\\n",
    "                                       ignore_idx=None, aoi=afghan_geom, cloud_filter=None)\n",
    "\n",
    "afghan_collection = mosaic_by_date(afghan_collection)\n",
    "print('number of images for afghan region', afghan_collection.size().getInfo())\n",
    "\n",
    "\n",
    "## execute data download\n",
    "# -- train\n",
    "# generate_array(afghan_collection, afghan_train_csv, os.path.join(path, 'data', 'Train_afghan'))\n",
    " \n",
    "# -- test\n",
    "# generate_array(afghan_collection, afghan_test_csv, os.path.join(path, 'data', 'Test_afghan'))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ab7c3c7-3fde-45c5-8710-bb214506788b",
   "metadata": {},
   "source": [
    "#### iran collection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f8227113-5a32-4e1f-a90a-fcedf942a76e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of images for iran region 55\n"
     ]
    }
   ],
   "source": [
    "# collection index to ignore\n",
    "iran_col_ignore = ['20', '47', '43',  '42', '39', '35', '34', '33', '31', '30', '29', '28', '26']\n",
    "\n",
    "iran_collection = prepare_collection(start_date='2019-07-01', end_date='2020-06-30', \\\n",
    "                                     ignore_idx=iran_col_ignore, aoi=iran_geom, cloud_filter=20)\n",
    "\n",
    "iran_collection = mosaic_by_date(iran_collection)\n",
    "print('number of images for iran region',iran_collection.size().getInfo())\n",
    "\n",
    "\n",
    "## execute data download\n",
    "# -- train\n",
    "# generate_array(iran_collection, iran_train_csv, os.path.join(path, 'data', 'Train_iran'))\n",
    "\n",
    "# -- test\n",
    "# generate_array(iran_collection, iran_test_csv, os.path.join(path, 'data', 'Test_iran'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0dda3ddb-ffce-4dbc-8200-b7bb4bc06e96",
   "metadata": {},
   "source": [
    "#### sudan collection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6b5bb580-91b3-40c3-9093-9663c6ddfc78",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of images for sudan region 51\n"
     ]
    }
   ],
   "source": [
    "# collection index to ignore\n",
    "sudan_col_ignore = ['45', '43', '50', '48', '49', '35', '2', '0']\n",
    "\n",
    "sudan_collection = prepare_collection(start_date='2019-07-01', end_date='2020-06-30', \\\n",
    "                                      ignore_idx=sudan_col_ignore, aoi=sudan_geom, cloud_filter=20)\n",
    "\n",
    "# sudan_collection = mosaic_by_date(sudan_collection)\n",
    "print('number of images for sudan region',sudan_collection.size().getInfo())\n",
    "\n",
    "\n",
    "## execute data download\n",
    "# -- train\n",
    "# generate_array(sudan_collection, sudan_train_csv, os.path.join(path, 'data', 'Train_sudan'))\n",
    "\n",
    "# --test\n",
    "# generate_array(sudan_collection, sudan_test_csv, os.path.join(path, 'data', 'Test_sudan'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "33b3ebc2-72ac-4245-8a0e-3c2093868703",
   "metadata": {},
   "source": [
    "## create master array per country (for raw bands and indices)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f6aa1e9-5477-4223-a67d-b5af59d227c3",
   "metadata": {},
   "source": [
    "This process collects all individual arrays in a folder and concatenates them\n",
    "- for each country, a master array is created for time series, labels, ids\n",
    "- time series has shape : number of samples x time x channels\n",
    "- labels has shape : number of samples. not available for Test\n",
    "- ids has shape : number of samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5c2b9dcd-4b1f-4270-9cc4-2a7c952972d3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "for afghan Train (500, 18, 10) (500, 18, 15)\n",
      "for afghan Test (500, 18, 10) (500, 18, 15)\n",
      "for sudan Train (500, 51, 10) (500, 51, 15)\n",
      "for sudan Test (500, 51, 10) (500, 51, 15)\n",
      "for iran Train (500, 55, 10) (500, 55, 15)\n",
      "for iran Test (500, 55, 10) (500, 55, 15)\n"
     ]
    }
   ],
   "source": [
    "list_country = ['afghan', 'sudan', 'iran']\n",
    "list_mode = ['Train', 'Test']\n",
    "\n",
    "for country in list_country:\n",
    "    for mode in list_mode:\n",
    "        \n",
    "        npy_path = 'data/{}_{}'.format(mode, country)\n",
    "        csv_path = 'data/{}.csv'.format(mode)\n",
    "        \n",
    "        if mode == 'Train':\n",
    "            X, y_labels, ids = create_ml_data(npy_path, csv_path)\n",
    "\n",
    "        if mode == 'Test':\n",
    "            csv_path = None\n",
    "            X, ids = create_ml_data(npy_path, csv_path)\n",
    "\n",
    "        X_indices = compute_indices(X)\n",
    "        print('for {} {}'.format(country, mode), X.shape, X_indices.shape)\n",
    "\n",
    "        # save master arraay of raw bands only and indices only\n",
    "        np.save(os.path.join('data', '{}_{}_raw.npy'.format(country, mode)), X)\n",
    "        np.save(os.path.join('data', '{}_{}_indices.npy'.format(country, mode)), X_indices)\n",
    "\n",
    "        # save y labels\n",
    "        if mode == 'Train':\n",
    "            np.save(os.path.join('data2', '{}_{}_labels.npy'.format(country, mode)), y_labels)\n",
    "\n",
    "        # save ids\n",
    "        np.save(os.path.join('data', '{}_{}_ids.npy'.format(country, mode)), ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b62125ec-18bb-4bad-9ca7-b118f2f6d8c6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}