notebooks/preprocess-dashboard.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af56e456-5fea-4e37-9e97-8a71f38036f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import xarray as xr\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50bf8081-d20b-4daf-bf8f-42fa410094bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = \"../workflows/pypsa-eur-sec/\"\n",
    "OUTPUT = \"../results/graphics-20221227/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f40ad2b-83b3-407f-9e5f-b993ab868c87",
   "metadata": {},
   "outputs": [],
   "source": [
    "# grid expansion, hydrogen network, costs, imports, shipping, onwind\n",
    "\n",
    "SCENARIOS = {\n",
    "    (\n",
    "        1,\n",
    "        1,\n",
    "        0,\n",
    "        0,\n",
    "        0,\n",
    "        0,\n",
    "    ): f\"{OUTPUT}/20221227-main/elec_s_181_lvopt__Co2L0-3H-T-H-B-I-A-solar+p3-linemaxext10_2050/\",\n",
    "    (\n",
    "        1,\n",
    "        1,\n",
    "        1,\n",
    "        0,\n",
    "        0,\n",
    "        0,\n",
    "    ): f\"{OUTPUT}/20221227-costs/elec_s_181_lvopt__Co2L0-3H-T-H-B-I-A-solar+p3-linemaxext10_2050/\",\n",
    "    (\n",
    "        1,\n",
    "        1,\n",
    "        0,\n",
    "        1,\n",
    "        0,\n",
    "        0,\n",
    "    ): f\"{OUTPUT}/20221227-import/elec_s_181_lvopt__Co2L0-3H-T-H-B-I-A-solar+p3-linemaxext10_2050/\",\n",
    "    (\n",
    "        1,\n",
    "        1,\n",
    "        0,\n",
    "        0,\n",
    "        1,\n",
    "        0,\n",
    "    ): f\"{OUTPUT}/20221227-shipping/elec_s_181_lvopt__Co2L0-3H-T-H-B-I-A-solar+p3-linemaxext10_2050/\",\n",
    "    (\n",
    "        1,\n",
    "        1,\n",
    "        0,\n",
    "        0,\n",
    "        0,\n",
    "        1,\n",
    "    ): f\"{OUTPUT}/20221227-main/elec_s_181_lvopt__Co2L0-3H-T-H-B-I-A-solar+p3-linemaxext10-onwind+p0_2050/\",\n",
    "    (\n",
    "        1,\n",
    "        0,\n",
    "        0,\n",
    "        0,\n",
    "        0,\n",
    "        0,\n",
    "    ): f\"{OUTPUT}/20221227-main/elec_s_181_lvopt__Co2L0-3H-T-H-B-I-A-solar+p3-linemaxext10-noH2network_2050/\",\n",
    "    (\n",
    "        1,\n",
    "        0,\n",
    "        1,\n",
    "        0,\n",
    "        0,\n",
    "        0,\n",
    "    ): f\"{OUTPUT}/20221227-costs/elec_s_181_lvopt__Co2L0-3H-T-H-B-I-A-solar+p3-linemaxext10-noH2network_2050/\",\n",
    "    (\n",
    "        1,\n",
    "        0,\n",
    "        0,\n",
    "        1,\n",
    "        0,\n",
    "        0,\n",
    "    ): f\"{OUTPUT}/20221227-import/elec_s_181_lvopt__Co2L0-3H-T-H-B-I-A-solar+p3-linemaxext10-noH2network_2050/\",\n",
    "    (\n",
    "        1,\n",
    "        0,\n",
    "        0,\n",
    "        0,\n",
    "        1,\n",
    "        0,\n",
    "    ): f\"{OUTPUT}/20221227-shipping/elec_s_181_lvopt__Co2L0-3H-T-H-B-I-A-solar+p3-linemaxext10-noH2network_2050/\",\n",
    "    (\n",
    "        1,\n",
    "        0,\n",
    "        0,\n",
    "        0,\n",
    "        0,\n",
    "        1,\n",
    "    ): f\"{OUTPUT}/20221227-main/elec_s_181_lvopt__Co2L0-3H-T-H-B-I-A-solar+p3-linemaxext10-noH2network-onwind+p0_2050/\",\n",
    "    (\n",
    "        0,\n",
    "        1,\n",
    "        0,\n",
    "        0,\n",
    "        0,\n",
    "        0,\n",
    "    ): f\"{OUTPUT}/20221227-main/elec_s_181_lv1.0__Co2L0-3H-T-H-B-I-A-solar+p3-linemaxext10_2050/\",\n",
    "    (\n",
    "        0,\n",
    "        1,\n",
    "        1,\n",
    "        0,\n",
    "        0,\n",
    "        0,\n",
    "    ): f\"{OUTPUT}/20221227-costs/elec_s_181_lv1.0__Co2L0-3H-T-H-B-I-A-solar+p3-linemaxext10_2050/\",\n",
    "    (\n",
    "        0,\n",
    "        1,\n",
    "        0,\n",
    "        1,\n",
    "        0,\n",
    "        0,\n",
    "    ): f\"{OUTPUT}/20221227-import/elec_s_181_lv1.0__Co2L0-3H-T-H-B-I-A-solar+p3-linemaxext10_2050/\",\n",
    "    (\n",
    "        0,\n",
    "        1,\n",
    "        0,\n",
    "        0,\n",
    "        1,\n",
    "        0,\n",
    "    ): f\"{OUTPUT}/20221227-shipping/elec_s_181_lv1.0__Co2L0-3H-T-H-B-I-A-solar+p3-linemaxext10_2050/\",\n",
    "    (\n",
    "        0,\n",
    "        1,\n",
    "        0,\n",
    "        0,\n",
    "        0,\n",
    "        1,\n",
    "    ): f\"{OUTPUT}/20221227-main/elec_s_181_lv1.0__Co2L0-3H-T-H-B-I-A-solar+p3-linemaxext10-onwind+p0_2050/\",\n",
    "    (\n",
    "        0,\n",
    "        0,\n",
    "        0,\n",
    "        0,\n",
    "        0,\n",
    "        0,\n",
    "    ): f\"{OUTPUT}/20221227-main/elec_s_181_lv1.0__Co2L0-3H-T-H-B-I-A-solar+p3-linemaxext10-noH2network_2050/\",\n",
    "    (\n",
    "        0,\n",
    "        0,\n",
    "        1,\n",
    "        0,\n",
    "        0,\n",
    "        0,\n",
    "    ): f\"{OUTPUT}/20221227-costs/elec_s_181_lv1.0__Co2L0-3H-T-H-B-I-A-solar+p3-linemaxext10-noH2network_2050/\",\n",
    "    (\n",
    "        0,\n",
    "        0,\n",
    "        0,\n",
    "        1,\n",
    "        0,\n",
    "        0,\n",
    "    ): f\"{OUTPUT}/20221227-import/elec_s_181_lv1.0__Co2L0-3H-T-H-B-I-A-solar+p3-linemaxext10-noH2network_2050/\",\n",
    "    (\n",
    "        0,\n",
    "        0,\n",
    "        0,\n",
    "        0,\n",
    "        1,\n",
    "        0,\n",
    "    ): f\"{OUTPUT}/20221227-shipping/elec_s_181_lv1.0__Co2L0-3H-T-H-B-I-A-solar+p3-linemaxext10-noH2network_2050/\",\n",
    "    (\n",
    "        0,\n",
    "        0,\n",
    "        0,\n",
    "        0,\n",
    "        0,\n",
    "        1,\n",
    "    ): f\"{OUTPUT}/20221227-main/elec_s_181_lv1.0__Co2L0-3H-T-H-B-I-A-solar+p3-linemaxext10-noH2network-onwind+p0_2050/\",\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4dd7d2e9-50ad-4455-b711-d542624e65d0",
   "metadata": {},
   "source": [
    "## Time Series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79598fcb-10c0-4adf-9230-5d186b1199ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets = [\n",
    "    \"total-electricity\",\n",
    "    \"total-heat\",\n",
    "    \"total-liquid-hydrocarbons\",\n",
    "    \"H2\",\n",
    "    \"gas\",\n",
    "    \"co2 stored\",\n",
    "    \"co2\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b627a86-c54b-4498-b50d-a1b218526c8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_vars = {}\n",
    "for item in datasets:\n",
    "    kwargs = dict(index_col=0, parse_dates=True)\n",
    "    ts = {\n",
    "        k: pd.read_csv(v + \"balance-ts-\" + item + \".csv\", **kwargs)\n",
    "        for k, v in SCENARIOS.items()\n",
    "    }\n",
    "\n",
    "    df = pd.concat(\n",
    "        ts,\n",
    "        names=[\n",
    "            \"power_grid\",\n",
    "            \"hydrogen_grid\",\n",
    "            \"optimistic_costs\",\n",
    "            \"imports\",\n",
    "            \"hydrogen_in_shipping\",\n",
    "            \"no_onwind\",\n",
    "        ],\n",
    "    )\n",
    "\n",
    "    df.columns.name = \"carrier\"\n",
    "\n",
    "    data_vars[item] = df.stack().to_xarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16b857ed-85b5-4278-97b5-88141ab845cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "ds = xr.Dataset(data_vars=data_vars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "650ce280-789c-456d-9f28-9911a4468e73",
   "metadata": {},
   "outputs": [],
   "source": [
    "comp = dict(zlib=True, complevel=9)\n",
    "encoding = {var: comp for var in ds.data_vars}\n",
    "ds.to_netcdf(\"time-series.nc\", encoding=encoding)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "96c3a9de-56aa-4b71-a02c-66219416022f",
   "metadata": {},
   "source": [
    "## Statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b3e8fe2-15c6-4802-9660-7870042dbe2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "kwargs = dict(index_col=0, header=[0, 1])\n",
    "ts = {k: pd.read_csv(v + \"report.csv\", **kwargs) for k, v in SCENARIOS.items()}\n",
    "\n",
    "df = pd.concat(\n",
    "    ts,\n",
    "    names=[\n",
    "        \"power_grid\",\n",
    "        \"hydrogen_grid\",\n",
    "        \"optimistic_costs\",\n",
    "        \"imports\",\n",
    "        \"hydrogen_in_shipping\",\n",
    "        \"no_onwind\",\n",
    "    ],\n",
    ").stack(level=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fea549d2-605b-45ee-8324-53e8d2dcd0f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.index.names = df.index.names[:-2] + [\"bus\", \"carrier\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01e1b099-c023-4af4-bdeb-4426715d0547",
   "metadata": {},
   "outputs": [],
   "source": [
    "ds = df.to_xarray()\n",
    "\n",
    "comp = dict(zlib=True, complevel=9)\n",
    "encoding = {var: comp for var in ds.data_vars}\n",
    "ds.to_netcdf(\"report.nc\", encoding=encoding)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac4e5736-c2cf-422f-8e51-cfdbbba8e224",
   "metadata": {},
   "source": [
    "## Electricity Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4adeaeb1-0067-4f27-bb23-8584f7b6f71b",
   "metadata": {},
   "outputs": [],
   "source": [
    "kwargs = dict(index_col=0, header=0)\n",
    "ts = {\n",
    "    k: pd.read_csv(v + \"edges-electricity.csv\", **kwargs) for k, v in SCENARIOS.items()\n",
    "}\n",
    "\n",
    "df = pd.concat(\n",
    "    ts,\n",
    "    names=[\n",
    "        \"power_grid\",\n",
    "        \"hydrogen_grid\",\n",
    "        \"optimistic_costs\",\n",
    "        \"imports\",\n",
    "        \"hydrogen_in_shipping\",\n",
    "        \"no_onwind\",\n",
    "    ],\n",
    ")\n",
    "df.index.names = df.index.names[:-1] + [\"line\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8edcae55-b970-4aba-b598-e70ad552c4b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "ds = df.to_xarray()\n",
    "\n",
    "comp = dict(zlib=True, complevel=9)\n",
    "encoding = {var: comp for var in ds.data_vars}\n",
    "ds.to_netcdf(\"electricity-network.nc\", encoding=encoding)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f5f5821-4869-4062-97b5-6ee2373c1034",
   "metadata": {},
   "source": [
    "## Hydrogen Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d8e98d53-9db1-4661-aa44-7c8c02e48b3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "kwargs = dict(index_col=0, header=0)\n",
    "ts = {\n",
    "    k: pd.read_csv(v + \"edges-hydrogen.csv\", **kwargs)\n",
    "    if os.path.exists(v + \"edges-hydrogen.csv\")\n",
    "    else None\n",
    "    for k, v in SCENARIOS.items()\n",
    "}\n",
    "\n",
    "df = pd.concat(\n",
    "    ts,\n",
    "    names=[\n",
    "        \"power_grid\",\n",
    "        \"hydrogen_grid\",\n",
    "        \"optimistic_costs\",\n",
    "        \"imports\",\n",
    "        \"hydrogen_in_shipping\",\n",
    "        \"no_onwind\",\n",
    "    ],\n",
    ")\n",
    "df.index.names = df.index.names[:-1] + [\"line\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b010b110-5ba0-4a94-b212-72d9a73c9115",
   "metadata": {},
   "outputs": [],
   "source": [
    "ds = df.to_xarray()\n",
    "\n",
    "comp = dict(zlib=True, complevel=9)\n",
    "encoding = {var: comp for var in ds.data_vars}\n",
    "ds.to_netcdf(\"hydrogen-network.nc\", encoding=encoding)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4c5e2bc6-9ee4-496d-8678-dbcfe3c77eae",
   "metadata": {},
   "source": [
    "## Sankey"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "273bde69-447c-40d5-89b1-e6b13ab1d0a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "ts = {\n",
    "    k: pd.read_csv(v + \"sankey.csv\", index_col=0).reset_index(drop=True)\n",
    "    for k, v in SCENARIOS.items()\n",
    "}\n",
    "\n",
    "df = pd.concat(\n",
    "    ts,\n",
    "    names=[\n",
    "        \"power_grid\",\n",
    "        \"hydrogen_grid\",\n",
    "        \"optimistic_costs\",\n",
    "        \"imports\",\n",
    "        \"hydrogen_in_shipping\",\n",
    "        \"no_onwind\",\n",
    "    ],\n",
    ")\n",
    "df.index.names = df.index.names[:-1] + [\"id\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f3c5f8c-f189-4aa8-a1a5-c6b2dfc85f6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "ds = df.to_xarray()\n",
    "\n",
    "comp = dict(zlib=True, complevel=9)\n",
    "encoding = {var: comp for var in ds.data_vars}\n",
    "ds.to_netcdf(\"energy-sankey.nc\", encoding=encoding)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed10ef47-27f1-4219-a214-53361506a020",
   "metadata": {},
   "source": [
    "## Carbon Sankey"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f852206c-8760-4eca-9d8e-fdf2fee358fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "ts = {\n",
    "    k: pd.read_csv(v + \"sankey-carbon.csv\", index_col=0).reset_index(drop=True)\n",
    "    for k, v in SCENARIOS.items()\n",
    "}\n",
    "\n",
    "df = pd.concat(\n",
    "    ts,\n",
    "    names=[\n",
    "        \"power_grid\",\n",
    "        \"hydrogen_grid\",\n",
    "        \"optimistic_costs\",\n",
    "        \"imports\",\n",
    "        \"hydrogen_in_shipping\",\n",
    "        \"no_onwind\",\n",
    "    ],\n",
    ")\n",
    "df.index.names = df.index.names[:-1] + [\"id\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d9fdfed-c3a6-49e6-b8bb-5af2179ba240",
   "metadata": {},
   "outputs": [],
   "source": [
    "ds = df.to_xarray()\n",
    "\n",
    "comp = dict(zlib=True, complevel=9)\n",
    "encoding = {var: comp for var in ds.data_vars}\n",
    "ds.to_netcdf(\"carbon-sankey.nc\", encoding=encoding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33b12564-b0b3-4d68-917a-587aa0e42616",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}