Skip to content

Commit

Permalink
Merge pull request #34 from worldbank/MENA
Browse files Browse the repository at this point in the history 
Mena
  • Loading branch information
@bpstewar
bpstewar authored Jan 21, 2025
2 parents d440073 + 3d54369 commit 8e91bf3
Show file tree
Hide file tree
Showing 9 changed files with 915 additions and 7,812 deletions.
7,711 changes: 10 additions & 7,701 deletions notebooks/Implementations/MENA_Benchmarking/FUA_ADMIN_Pop.ipynb
View file

Large diffs are not rendered by default.

212 changes: 212 additions & 0 deletions notebooks/Implementations/MENA_Benchmarking/GHSL_POP_zonal_stats.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,212 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "0",
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"import os\n",
"import rasterio\n",
"\n",
"import geopandas as gpd\n",
"import pandas as pd\n",
"\n",
"sys.path.append(\"/home/wb411133/Code/GOSTrocks/src\")\n",
"\n",
"import GOSTrocks.rasterMisc as rMisc\n",
"from GOSTrocks.misc import tPrint"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1",
"metadata": {},
"outputs": [],
"source": [
"data_folder = \"s3://wbg-geography01/URBANIZATION/MENA/Extents/\"\n",
"ucdb_file = \"/home/wb411133/Code/GOSTurban/GHS_STAT_UCDB2015MT_GLOBE_R2019A_V1_2.gpkg\"\n",
"fua_file = os.path.join(data_folder, \"GHS_FUA_UCDB2015_GLOBE_R2019A_54009_1K_V1_0.gpkg\")\n",
"fua_peripheries = os.path.join(data_folder, \"FUA_peripheries.gpkg\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2",
"metadata": {},
"outputs": [],
"source": [
"inU = gpd.read_file(ucdb_file)\n",
"# If the peripheries exists read them in, if not, create them\n",
"inF = gpd.read_file(fua_file)\n",
"inF = inF.to_crs(inU.crs)\n",
"\n",
"\"\"\"inP = gpd.read_file(fua_peripheries)\n",
"inP = inP.to_crs(inU.crs)\n",
"inP['geometry'] = inP.buffer(0)\n",
"\"\"\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3",
"metadata": {},
"outputs": [],
"source": [
"ghsl_folder = \"/home/public/Data/GLOBAL/GHSL/Built\"\n",
"ghsl_files = [x for x in os.listdir(ghsl_folder) if x.endswith(\".tif\")]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4",
"metadata": {},
"outputs": [],
"source": [
"out_folder = \"s3://wbg-geography01/URBANIZATION/MENA/ZONAL_RES/GHSL\"\n",
"fua_res_file = os.path.join(out_folder, \"fua_ntl_zonal.csv\")\n",
"core_res_file = os.path.join(out_folder, \"core_ntl_zonal.csv\")\n",
"\n",
"fua_zonal = inF.copy()\n",
"core_zonal = inU.copy()\n",
"for ghsl_file in ghsl_files:\n",
" year = ghsl_file.split(\"_\")[3][1:]\n",
" inR = rasterio.open(os.path.join(ghsl_folder, ghsl_file))\n",
" if inF.crs != inR.crs:\n",
" inF = inF.to_crs(inR.crs)\n",
" tPrint(\"Reprojected FUAs\")\n",
" if inU.crs != inR.crs:\n",
" inU = inU.to_crs(inR.crs)\n",
" tPrint(\"Reprojected Cores\")\n",
" # Run zonal on FUA\n",
" fua_res = rMisc.zonalStats(inF, inR, minVal=0, maxVal=10000)\n",
" fua_res = pd.DataFrame(fua_res, columns=[\"SUM\", \"MIN\", \"MAX\", \"MEAN\"])\n",
" fua_zonal[f\"ghsl_{year}\"] = fua_res[\"SUM\"]\n",
" # Run zonal on core\n",
" core_res = rMisc.zonalStats(inU, inR, minVal=0, maxVal=10000)\n",
" core_res = pd.DataFrame(fua_res, columns=[\"SUM\", \"MIN\", \"MAX\", \"MEAN\"])\n",
" core_zonal[f\"ghsl_{year}\"] = core_res[\"SUM\"]\n",
" tPrint(f\"Completed {year}\")\n",
"core_zonal.to_csv(core_res_file)\n",
"fua_zonal.to_csv(fua_res_file)"
]
},
{
"cell_type": "markdown",
"id": "5",
"metadata": {},
"source": [
"# Population stats"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6",
"metadata": {},
"outputs": [],
"source": [
"ghs_pop_folder = \"/home/public/Data/GLOBAL/GHSL/Pop\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7",
"metadata": {},
"outputs": [],
"source": [
"\"\"\"# Downlaod and unzip GHS_POP data\n",
"import urllib.request\n",
"import zipfile\n",
"url_path_base = \"https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/GHSL/GHS_POP_GLOBE_R2023A/GHS_POP_E{year}_GLOBE_R2023A_54009_1000/V1-0/GHS_POP_E{year}_GLOBE_R2023A_54009_1000_V1_0.zip\"\n",
"for year in range(1975, 2021, 5):\n",
" url_path = url_path_base.format(year=year) \n",
" out_file = os.path.join(ghs_pop_folder, os.path.basename(url_path))\n",
" if not os.path.exists(out_file)\n",
" urllib.request.urlretrieve(url_path, out_file)\n",
" tPrint(out_file)\n",
"zip_files = [x for x in os.listdir(ghs_pop_folder) if x.endswith(\"0.zip\")]\n",
"for zip_file in zip_files:\n",
" with zipfile.ZipFile(os.path.join(ghs_pop_folder, zip_file), 'r') as zip_ref:\n",
" zip_ref.extractall(ghs_pop_folder)\n",
" tPrint(zip_file)\n",
"for zip_file in zip_files:\n",
" os.remove(os.path.join(ghs_pop_folder, zip_file))\n",
" \n",
"\"\"\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8",
"metadata": {},
"outputs": [],
"source": [
"out_folder = \"s3://wbg-geography01/URBANIZATION/MENA/ZONAL_RES/GHSPop\"\n",
"fua_res_file = os.path.join(out_folder, \"fua_ghspop_zonal.csv\")\n",
"core_res_file = os.path.join(out_folder, \"core_ghspop_zonal.csv\")\n",
"ghs_pop_files = [x for x in os.listdir(ghs_pop_folder) if x.endswith(\"1000_V1_0.tif\")]\n",
"\n",
"fua_zonal = inF.copy()\n",
"core_zonal = inU.copy()\n",
"for ghs_pop_file in ghs_pop_files:\n",
" year = ghs_pop_file.split(\"_\")[2][1:]\n",
" inR = rasterio.open(os.path.join(ghs_pop_folder, ghs_pop_file))\n",
" if inF.crs != inR.crs:\n",
" inF = inF.to_crs(inR.crs)\n",
" tPrint(\"Reprojected FUAs\")\n",
" if inU.crs != inR.crs:\n",
" inU = inU.to_crs(inR.crs)\n",
" tPrint(\"Reprojected Cores\")\n",
" # Run zonal on FUA\n",
" fua_res = rMisc.zonalStats(inF, inR, minVal=0)\n",
" fua_res = pd.DataFrame(fua_res, columns=[\"SUM\", \"MIN\", \"MAX\", \"MEAN\"])\n",
" fua_zonal[f\"ghs_pop_{year}\"] = fua_res[\"SUM\"]\n",
" # Run zonal on core\n",
" core_res = rMisc.zonalStats(inU, inR, minVal=0, maxVal=10000)\n",
" core_res = pd.DataFrame(fua_res, columns=[\"SUM\", \"MIN\", \"MAX\", \"MEAN\"])\n",
" core_zonal[f\"ghs_pop_{year}\"] = core_res[\"SUM\"]\n",
" tPrint(f\"Completed {year}\")\n",
"core_zonal.to_csv(core_res_file)\n",
"fua_zonal.to_csv(fua_res_file)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Earth Engine",
"language": "python",
"name": "ee"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.4"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Loading

0 comments on commit 8e91bf3

Please sign in to comment.