diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9b26b19..0663a7d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,16 +30,17 @@ repos: rev: v1.5.1 hooks: - id: docformatter - args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"] + args: ["--in-place", "--make-summary-multi-line"] - repo: https://github.com/keewis/blackdoc rev: v0.3.8 hooks: - id: blackdoc + additional_dependencies: ["black==23.3.0"] # Formatting with "black" coding style - repo: https://github.com/psf/black - rev: 23.1.0 + rev: 23.3.0 hooks: # Format Python files - id: black diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7704292..39b8bcb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,7 +3,14 @@ Changelog ========= -* :py:func:`~core.describelevel` superseedes the as-of-now deprecated +v0.2.5 (2023-05-04) +------------------------------------------------------------ +* :func:`~core.formatlevel` and :func:`~core.extractlevel` (or their equivalents + :meth:`~accessors.DataFrameIdxAccessor.format` and + :meth:`~accessors.DataFrameIdxAccessor.extract`) make it easy to combine or split + index levels using format-string like templates; check examples in the guide + (:ref:`Selecting data`) :pull:`13` +* :py:func:`~core.describelevel` superseeds the as-of-now deprecated :py:func:`~core.summarylevel` :pull:`11` v0.2.4 (2023-05-03) diff --git a/docs/conf.py b/docs/conf.py index 6771b1d..fda9fd4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -36,6 +36,7 @@ "sphinx.ext.coverage", "sphinx.ext.doctest", "sphinx.ext.extlinks", + "sphinx.ext.autosectionlabel", # "sphinx.ext.ifconfig", "sphinx.ext.napoleon", "sphinx.ext.intersphinx", diff --git a/docs/notebooks/introduction.ipynb b/docs/notebooks/introduction.ipynb index 557ebff..a924851 100644 --- a/docs/notebooks/introduction.ipynb +++ b/docs/notebooks/introduction.ipynb @@ -1,14 +1,5 @@ { "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "id": "8df2b6a3-ceed-40ca-9908-a92e55551a38", - "metadata": {}, - "source": [ - "# Introduction" - ] - }, { "attachments": {}, "cell_type": "markdown", @@ -34,7 +25,7 @@ "id": "814f4bb8-be13-47cd-9853-eb23f61a1da3", "metadata": {}, "source": [ - "## Test data set\n", + "# Test data set\n", "\n", "For experimenting and easy testing `pandas-indexing` brings along the power sector generation and capacity of the HighRE illustrative modelling pathway from the IPCC AR6 scenario database in IAMC format." ] @@ -361,6 +352,30 @@ "df.head()" ] }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "0d61c065", + "metadata": {}, + "source": [ + "# Usage styles\n", + "\n", + "`pandas-indexing` defines two different usage styles:\n", + "There are:\n", + "1. functions that can be imported from the toplevel module, like\n", + " ```python\n", + " from pandas_indexing import assignlevel\n", + " assignlevel(df, unit=\"Mt CO2e/yr\")\n", + " ```\n", + "2. convenience accessors that are hooking into pandas as extensions\n", + " ```python\n", + " import pandas_indexing.accessors\n", + " df.idx.assign(unit=\"Mt CO2e/yr)\n", + " ```\n", + "\n", + "Most of the functionality is available with both styles under slightly different names. I'll present the functional style here first (and add the alternative as comments)" + ] + }, { "cell_type": "code", "execution_count": 3, @@ -371,17 +386,53 @@ "name": "stdout", "output_type": "stream", "text": [ - "model: REMIND-MAgPIE 2.1-4.3\n", - "scenario: DeepElec_SSP2_HighRE_Budg900\n", - "region: World\n", - "variable: Capacity|Electricity|Biomass, Capacity|Electricity|Coal, Capacity|Electricity|Gas, Capacity|Electricity|Geothermal, Capacity|Electricity|Hydro, Capacity|Electricity|Nuclear, Capacity|Electricity|Oil, Capacity|Electricity|Other, Capacity|Electricity|Solar, Capacity|Electricity|Wind, Secondary Energy|Electricity|Biomass, Secondary Energy|Electricity|Coal, Secondary Energy|Electricity|Gas, Secondary Energy|Electricity|Geothermal, Secondary Energy|Electricity|Hydro, Secondary Energy|Electricity|Nuclear, Secondary Energy|Electricity|Oil, Secondary Energy|Electricity|Other, Secondary Energy|Electricity|Solar, Secondary Energy|Electricity|Wind\n", - "unit: GW, GWh/yr\n" + "Index:\n", + " * model : REMIND-MAgPIE 2.1-4.3 (1)\n", + " * scenario : DeepElec_SSP2_HighRE_Budg900 (1)\n", + " * region : World (1)\n", + " * variable : Capacity|Electricity|Biomass, ... (20)\n", + " * unit : GW, GWh/yr (2)\n", + "\n", + "Columns:\n", + " * : 2005, 2010, 2015, 2020, 2025, 2030, 2035, 2040, ... 2100 (16)\n" ] } ], "source": [ - "for name in df.index.names:\n", - " print(f\"{name}: {', '.join(df.index.unique(name))}\")" + "from pandas_indexing.core import describelevel\n", + "\n", + "describelevel(df) # or: df.idx" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "a4f23678", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Index:\n", + " * model : REMIND-MAgPIE 2.1-4.3 (1)\n", + " * scenario : DeepElec_SSP2_HighRE_Budg900 (1)\n", + " * region : World (1)\n", + " * variable : Capacity|Electricity|Biomass, ... (20)\n", + " * unit : GW, GWh/yr (2)\n", + "\n", + "Columns:\n", + " * : 2005, 2010, 2015, 2020, 2025, 2030, 2035, 2040, ... 2100 (16)" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas_indexing.accessors\n", + "\n", + "df.idx" ] }, { @@ -401,7 +452,7 @@ "id": "61d70ee2", "metadata": {}, "source": [ - "## Selecting data\n", + "# Selecting data\n", "\n", "For using pandas indexes effectively for computations, it makes sense to split the hierarchically variable index out into separate python variables: `generation` and `capacity`. The standard pandas tools for this job are `pd.DataFrame.loc` in conjunction with `pd.IndexSlice` or `pd.DataFrame.query`. \n", "\n", @@ -410,17 +461,17 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "id": "36d58553", "metadata": {}, "outputs": [], "source": [ - "from pandas_indexing import ismatch, isin" + "from pandas_indexing import ismatch, isin # no .idx equivalents" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "id": "982b9663", "metadata": {}, "outputs": [ @@ -441,7 +492,7 @@ "Name: 2030, dtype: float64" ] }, - "execution_count": 5, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -461,7 +512,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "id": "711f2683", "metadata": {}, "outputs": [ @@ -949,7 +1000,7 @@ " Wind GW 17368.8215 " ] }, - "execution_count": 6, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -967,16 +1018,16 @@ { "attachments": {}, "cell_type": "markdown", - "id": "e78ace7d", + "id": "3087eb3c", "metadata": {}, "source": [ - "So that we can easily calculate capacity factors (ratios of generation and capacity) directly, if we take care of removing the conflicting `unit` level. Similarly to `ismatch`, `isin` can be provided as an argument to `.loc[]` to select on named index levels with the difference that only exact matches are considered." + "Since this extraction of data is relatively common, `extractlevel` simplifies this by matching against a format-like template string:" ] }, { "cell_type": "code", - "execution_count": 7, - "id": "9b4be2b0", + "execution_count": 8, + "id": "16032dd2", "metadata": {}, "outputs": [ { @@ -1003,17 +1054,43 @@ " \n", " \n", " \n", + " \n", + " \n", + " 2005\n", + " 2010\n", + " 2015\n", + " 2020\n", + " 2025\n", " 2030\n", " 2035\n", " 2040\n", " 2045\n", " 2050\n", + " 2055\n", + " 2060\n", + " 2070\n", + " 2080\n", + " 2090\n", + " 2100\n", " \n", " \n", " model\n", " scenario\n", " region\n", - " variable\n", + " unit\n", + " carrier\n", + " fuel\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", @@ -1023,110 +1100,412 @@ " \n", " \n", " \n", - " REMIND-MAgPIE 2.1-4.3\n", - " DeepElec_SSP2_HighRE_Budg900\n", - " World\n", + " REMIND-MAgPIE 2.1-4.3\n", + " DeepElec_SSP2_HighRE_Budg900\n", + " World\n", + " GWh/yr\n", + " Electricity\n", + " Biomass\n", + " 2.352778e+05\n", + " 3.407222e+05\n", + " 4.669444e+05\n", + " 5.942778e+05\n", + " 7.219167e+05\n", + " 8.496389e+05\n", + " 9.855833e+05\n", + " 1.173833e+06\n", + " 1.359639e+06\n", + " 1.471139e+06\n", + " 1.513056e+06\n", + " 1.544750e+06\n", + " 1.564583e+06\n", + " 1.557556e+06\n", + " 1.543056e+06\n", + " 1.574306e+06\n", + " \n", + " \n", + " Coal\n", + " 7.419778e+06\n", + " 8.252667e+06\n", + " 9.246333e+06\n", + " 1.075747e+07\n", + " 6.045972e+06\n", + " 8.733611e+05\n", + " 6.444444e+03\n", + " 5.027778e+03\n", + " 3.694444e+03\n", + " 2.416667e+03\n", + " 1.555556e+03\n", + " 9.444444e+02\n", + " 4.444444e+02\n", + " 5.277778e+02\n", + " 5.555556e+02\n", + " 5.000000e+02\n", + " \n", + " \n", + " Gas\n", + " 3.816028e+06\n", + " 5.050500e+06\n", + " 5.968778e+06\n", + " 7.060083e+06\n", + " 6.469250e+06\n", + " 4.551528e+06\n", + " 2.907250e+06\n", + " 2.602278e+06\n", + " 1.752056e+06\n", + " 1.218361e+06\n", + " 6.538889e+05\n", + " 5.281944e+05\n", + " 2.423333e+05\n", + " 1.685833e+05\n", + " 4.166667e+04\n", + " 3.888889e+02\n", + " \n", + " \n", " Geothermal\n", - " 0.849984\n", - " 0.850005\n", - " 0.849989\n", - " 0.849986\n", - " 0.850021\n", + " 6.041667e+04\n", + " 9.677778e+04\n", + " 1.799167e+05\n", + " 3.474167e+05\n", + " 5.260278e+05\n", + " 6.213611e+05\n", + " 6.291111e+05\n", + " 6.177778e+05\n", + " 5.893056e+05\n", + " 5.187222e+05\n", + " 4.333611e+05\n", + " 3.603889e+05\n", + " 2.924722e+05\n", + " 2.492778e+05\n", + " 2.059167e+05\n", + " 1.937500e+05\n", " \n", " \n", " Hydro\n", - " 0.394597\n", - " 0.394474\n", - " 0.396240\n", - " 0.398217\n", - " 0.399716\n", + " 2.976944e+06\n", + " 3.557028e+06\n", + " 4.386694e+06\n", + " 5.302056e+06\n", + " 5.813167e+06\n", + " 6.173778e+06\n", + " 6.423194e+06\n", + " 6.604167e+06\n", + " 6.717417e+06\n", + " 6.770250e+06\n", + " 6.782028e+06\n", + " 6.775917e+06\n", + " 6.722667e+06\n", + " 6.655833e+06\n", + " 6.631806e+06\n", + " 6.533167e+06\n", " \n", " \n", - " Solar\n", - " 0.161156\n", - " 0.154237\n", - " 0.151994\n", - " 0.150698\n", - " 0.149422\n", + " Nuclear\n", + " 2.861861e+06\n", + " 2.642750e+06\n", + " 2.448556e+06\n", + " 2.214167e+06\n", + " 2.055667e+06\n", + " 1.876222e+06\n", + " 1.617278e+06\n", + " 1.502778e+06\n", + " 1.304083e+06\n", + " 1.098694e+06\n", + " 8.686111e+05\n", + " 6.451944e+05\n", + " 2.717500e+05\n", + " 5.925000e+04\n", + " 3.916667e+03\n", + " 1.388889e+02\n", " \n", " \n", - " Wind\n", - " 0.263820\n", - " 0.275781\n", - " 0.289083\n", - " 0.302018\n", - " 0.298163\n", + " Oil\n", + " 1.088861e+06\n", + " 1.016972e+06\n", + " 8.353889e+05\n", + " 6.174167e+05\n", + " 4.317778e+05\n", + " 1.960556e+05\n", + " 6.919444e+04\n", + " 4.797222e+04\n", + " 3.175000e+04\n", + " 1.169444e+04\n", + " 6.722222e+03\n", + " 2.777778e+01\n", + " 2.777778e+01\n", + " 2.777778e+01\n", + " 2.777778e+01\n", + " 2.777778e+01\n", " \n", - " \n", - "\n", - "" - ], - "text/plain": [ - " 2030 \\\n", - "model scenario region variable \n", - "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Geothermal 0.849984 \n", - " Hydro 0.394597 \n", - " Solar 0.161156 \n", - " Wind 0.263820 \n", - "\n", - " 2035 \\\n", - "model scenario region variable \n", - "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Geothermal 0.850005 \n", - " Hydro 0.394474 \n", - " Solar 0.154237 \n", - " Wind 0.275781 \n", - "\n", - " 2040 \\\n", - "model scenario region variable \n", - "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Geothermal 0.849989 \n", - " Hydro 0.396240 \n", - " Solar 0.151994 \n", - " Wind 0.289083 \n", - "\n", - " 2045 \\\n", - "model scenario region variable \n", - "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Geothermal 0.849986 \n", - " Hydro 0.398217 \n", - " Solar 0.150698 \n", - " Wind 0.302018 \n", - "\n", - " 2050 \n", - "model scenario region variable \n", - "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Geothermal 0.850021 \n", - " Hydro 0.399716 \n", - " Solar 0.149422 \n", - " Wind 0.298163 " - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "capacity_factor = generation.droplevel(\"unit\") / 8760 / capacity.droplevel(\"unit\")\n", - "capacity_factor.loc[isin(variable=[\"Solar\", \"Wind\", \"Hydro\", \"Geothermal\"]), 2030:2051]" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "b4d501a3", - "metadata": {}, - "source": [ - "Under the hood `isin` and `ismatch` generate `Selector` objects that can be called with data to generate boolean masks. They can be composed into complex queries intuitively, which are kept as a hierarchical structure of objects." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "4197be7d", - "metadata": {}, - "outputs": [ - { - "data": { + " \n", + " Other\n", + " 0.000000e+00\n", + " 0.000000e+00\n", + " 4.444444e+02\n", + " 2.166667e+03\n", + " 8.913889e+04\n", + " 4.938889e+05\n", + " 1.204583e+06\n", + " 1.985111e+06\n", + " 2.790639e+06\n", + " 3.488083e+06\n", + " 4.020806e+06\n", + " 4.434639e+06\n", + " 5.261472e+06\n", + " 6.099583e+06\n", + " 6.740833e+06\n", + " 7.547194e+06\n", + " \n", + " \n", + " Solar\n", + " 4.083333e+03\n", + " 8.325000e+04\n", + " 2.616111e+05\n", + " 9.263611e+05\n", + " 5.500722e+06\n", + " 1.566242e+07\n", + " 2.575731e+07\n", + " 3.404253e+07\n", + " 4.088394e+07\n", + " 4.723214e+07\n", + " 5.327061e+07\n", + " 5.819236e+07\n", + " 6.826367e+07\n", + " 7.754106e+07\n", + " 8.359456e+07\n", + " 9.270394e+07\n", + " \n", + " \n", + " Wind\n", + " 1.091389e+05\n", + " 5.946944e+05\n", + " 8.323889e+05\n", + " 1.482806e+06\n", + " 3.695778e+06\n", + " 8.141417e+06\n", + " 1.318953e+07\n", + " 1.727517e+07\n", + " 2.112919e+07\n", + " 2.367689e+07\n", + " 2.569986e+07\n", + " 2.771675e+07\n", + " 3.280081e+07\n", + " 3.570786e+07\n", + " 3.840797e+07\n", + " 4.123369e+07\n", + " \n", + " \n", + "\n", + "" + ], "text/plain": [ - "And(a=Isin(filters={'variable': ['Coal', 'Gas', 'Nuclear'], 'unit': 'GW'}), b=Not(a=Ismatch(filters={'variable': 'S*'}, regex=False)))" + " 2005 \\\n", + "model scenario region unit carrier fuel \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Electricity Biomass 2.352778e+05 \n", + " Coal 7.419778e+06 \n", + " Gas 3.816028e+06 \n", + " Geothermal 6.041667e+04 \n", + " Hydro 2.976944e+06 \n", + " Nuclear 2.861861e+06 \n", + " Oil 1.088861e+06 \n", + " Other 0.000000e+00 \n", + " Solar 4.083333e+03 \n", + " Wind 1.091389e+05 \n", + "\n", + " 2010 \\\n", + "model scenario region unit carrier fuel \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Electricity Biomass 3.407222e+05 \n", + " Coal 8.252667e+06 \n", + " Gas 5.050500e+06 \n", + " Geothermal 9.677778e+04 \n", + " Hydro 3.557028e+06 \n", + " Nuclear 2.642750e+06 \n", + " Oil 1.016972e+06 \n", + " Other 0.000000e+00 \n", + " Solar 8.325000e+04 \n", + " Wind 5.946944e+05 \n", + "\n", + " 2015 \\\n", + "model scenario region unit carrier fuel \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Electricity Biomass 4.669444e+05 \n", + " Coal 9.246333e+06 \n", + " Gas 5.968778e+06 \n", + " Geothermal 1.799167e+05 \n", + " Hydro 4.386694e+06 \n", + " Nuclear 2.448556e+06 \n", + " Oil 8.353889e+05 \n", + " Other 4.444444e+02 \n", + " Solar 2.616111e+05 \n", + " Wind 8.323889e+05 \n", + "\n", + " 2020 \\\n", + "model scenario region unit carrier fuel \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Electricity Biomass 5.942778e+05 \n", + " Coal 1.075747e+07 \n", + " Gas 7.060083e+06 \n", + " Geothermal 3.474167e+05 \n", + " Hydro 5.302056e+06 \n", + " Nuclear 2.214167e+06 \n", + " Oil 6.174167e+05 \n", + " Other 2.166667e+03 \n", + " Solar 9.263611e+05 \n", + " Wind 1.482806e+06 \n", + "\n", + " 2025 \\\n", + "model scenario region unit carrier fuel \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Electricity Biomass 7.219167e+05 \n", + " Coal 6.045972e+06 \n", + " Gas 6.469250e+06 \n", + " Geothermal 5.260278e+05 \n", + " Hydro 5.813167e+06 \n", + " Nuclear 2.055667e+06 \n", + " Oil 4.317778e+05 \n", + " Other 8.913889e+04 \n", + " Solar 5.500722e+06 \n", + " Wind 3.695778e+06 \n", + "\n", + " 2030 \\\n", + "model scenario region unit carrier fuel \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Electricity Biomass 8.496389e+05 \n", + " Coal 8.733611e+05 \n", + " Gas 4.551528e+06 \n", + " Geothermal 6.213611e+05 \n", + " Hydro 6.173778e+06 \n", + " Nuclear 1.876222e+06 \n", + " Oil 1.960556e+05 \n", + " Other 4.938889e+05 \n", + " Solar 1.566242e+07 \n", + " Wind 8.141417e+06 \n", + "\n", + " 2035 \\\n", + "model scenario region unit carrier fuel \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Electricity Biomass 9.855833e+05 \n", + " Coal 6.444444e+03 \n", + " Gas 2.907250e+06 \n", + " Geothermal 6.291111e+05 \n", + " Hydro 6.423194e+06 \n", + " Nuclear 1.617278e+06 \n", + " Oil 6.919444e+04 \n", + " Other 1.204583e+06 \n", + " Solar 2.575731e+07 \n", + " Wind 1.318953e+07 \n", + "\n", + " 2040 \\\n", + "model scenario region unit carrier fuel \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Electricity Biomass 1.173833e+06 \n", + " Coal 5.027778e+03 \n", + " Gas 2.602278e+06 \n", + " Geothermal 6.177778e+05 \n", + " Hydro 6.604167e+06 \n", + " Nuclear 1.502778e+06 \n", + " Oil 4.797222e+04 \n", + " Other 1.985111e+06 \n", + " Solar 3.404253e+07 \n", + " Wind 1.727517e+07 \n", + "\n", + " 2045 \\\n", + "model scenario region unit carrier fuel \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Electricity Biomass 1.359639e+06 \n", + " Coal 3.694444e+03 \n", + " Gas 1.752056e+06 \n", + " Geothermal 5.893056e+05 \n", + " Hydro 6.717417e+06 \n", + " Nuclear 1.304083e+06 \n", + " Oil 3.175000e+04 \n", + " Other 2.790639e+06 \n", + " Solar 4.088394e+07 \n", + " Wind 2.112919e+07 \n", + "\n", + " 2050 \\\n", + "model scenario region unit carrier fuel \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Electricity Biomass 1.471139e+06 \n", + " Coal 2.416667e+03 \n", + " Gas 1.218361e+06 \n", + " Geothermal 5.187222e+05 \n", + " Hydro 6.770250e+06 \n", + " Nuclear 1.098694e+06 \n", + " Oil 1.169444e+04 \n", + " Other 3.488083e+06 \n", + " Solar 4.723214e+07 \n", + " Wind 2.367689e+07 \n", + "\n", + " 2055 \\\n", + "model scenario region unit carrier fuel \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Electricity Biomass 1.513056e+06 \n", + " Coal 1.555556e+03 \n", + " Gas 6.538889e+05 \n", + " Geothermal 4.333611e+05 \n", + " Hydro 6.782028e+06 \n", + " Nuclear 8.686111e+05 \n", + " Oil 6.722222e+03 \n", + " Other 4.020806e+06 \n", + " Solar 5.327061e+07 \n", + " Wind 2.569986e+07 \n", + "\n", + " 2060 \\\n", + "model scenario region unit carrier fuel \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Electricity Biomass 1.544750e+06 \n", + " Coal 9.444444e+02 \n", + " Gas 5.281944e+05 \n", + " Geothermal 3.603889e+05 \n", + " Hydro 6.775917e+06 \n", + " Nuclear 6.451944e+05 \n", + " Oil 2.777778e+01 \n", + " Other 4.434639e+06 \n", + " Solar 5.819236e+07 \n", + " Wind 2.771675e+07 \n", + "\n", + " 2070 \\\n", + "model scenario region unit carrier fuel \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Electricity Biomass 1.564583e+06 \n", + " Coal 4.444444e+02 \n", + " Gas 2.423333e+05 \n", + " Geothermal 2.924722e+05 \n", + " Hydro 6.722667e+06 \n", + " Nuclear 2.717500e+05 \n", + " Oil 2.777778e+01 \n", + " Other 5.261472e+06 \n", + " Solar 6.826367e+07 \n", + " Wind 3.280081e+07 \n", + "\n", + " 2080 \\\n", + "model scenario region unit carrier fuel \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Electricity Biomass 1.557556e+06 \n", + " Coal 5.277778e+02 \n", + " Gas 1.685833e+05 \n", + " Geothermal 2.492778e+05 \n", + " Hydro 6.655833e+06 \n", + " Nuclear 5.925000e+04 \n", + " Oil 2.777778e+01 \n", + " Other 6.099583e+06 \n", + " Solar 7.754106e+07 \n", + " Wind 3.570786e+07 \n", + "\n", + " 2090 \\\n", + "model scenario region unit carrier fuel \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Electricity Biomass 1.543056e+06 \n", + " Coal 5.555556e+02 \n", + " Gas 4.166667e+04 \n", + " Geothermal 2.059167e+05 \n", + " Hydro 6.631806e+06 \n", + " Nuclear 3.916667e+03 \n", + " Oil 2.777778e+01 \n", + " Other 6.740833e+06 \n", + " Solar 8.359456e+07 \n", + " Wind 3.840797e+07 \n", + "\n", + " 2100 \n", + "model scenario region unit carrier fuel \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Electricity Biomass 1.574306e+06 \n", + " Coal 5.000000e+02 \n", + " Gas 3.888889e+02 \n", + " Geothermal 1.937500e+05 \n", + " Hydro 6.533167e+06 \n", + " Nuclear 1.388889e+02 \n", + " Oil 2.777778e+01 \n", + " Other 7.547194e+06 \n", + " Solar 9.270394e+07 \n", + " Wind 4.123369e+07 " ] }, "execution_count": 8, @@ -1135,78 +1514,26 @@ } ], "source": [ - "query = isin(variable=[\"Coal\", \"Gas\", \"Nuclear\"], unit=\"GW\") & ~ismatch(variable=\"S*\")\n", - "query" + "from pandas_indexing import extractlevel, formatlevel\n", + "\n", + "splitdf = extractlevel(df, variable=\"Secondary Energy|{carrier}|{fuel}\", drop=True)\n", + "# or: df.idx.extract(variable=\"Secondary Energy|{carrier}|{fuel}\")\n", + "splitdf" ] }, { "attachments": {}, "cell_type": "markdown", - "id": "f137127f", + "id": "405720c7", "metadata": {}, "source": [ - "For evaluating the query one needs to pass in a data object to produce a boolean mask. Since pandas `.loc` indexer does exactly that, these queries work as expected.\n" + "The inverse operation is to combine strings back together with `formatlevel`:" ] }, { "cell_type": "code", "execution_count": 9, - "id": "18401aa0", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "model scenario region variable unit\n", - "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Biomass GW False\n", - " Coal GW True\n", - " Gas GW True\n", - " Geothermal GW False\n", - " Hydro GW False\n", - " Nuclear GW True\n", - " Oil GW False\n", - " Other GW False\n", - " Solar GW False\n", - " Wind GW False\n", - "dtype: bool" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "query(capacity)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "id": "7e1d4f00", - "metadata": {}, - "source": [ - "
\n", - "\n", - "WARNING\n", - "\n", - "It is currently impossible to use a pandas boolean series **in front of** a selector; ie.\n", - "\n", - "```python\n", - "(capacity[2030] > 250) & isin(variable=[\"Coal\", \"Gas\", \"Nuclear\"], unit=\"GW\")\n", - "```\n", - "will **fail**, it needs to be\n", - "```python\n", - "isin(variable=[\"Coal\", \"Gas\", \"Nuclear\"], unit=\"GW\") & (capacity[2030] > 250)\n", - "```\n", - "\n", - "
" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "8b421dba", + "id": "8e355684", "metadata": {}, "outputs": [ { @@ -1242,13 +1569,29 @@ " 2030\n", " 2035\n", " 2040\n", + " 2045\n", + " 2050\n", + " 2055\n", + " 2060\n", + " 2070\n", + " 2080\n", + " 2090\n", + " 2100\n", " \n", " \n", " model\n", " scenario\n", " region\n", - " variable\n", " unit\n", + " variable\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", @@ -1261,104 +1604,436 @@ " \n", " \n", " \n", - " REMIND-MAgPIE 2.1-4.3\n", - " DeepElec_SSP2_HighRE_Budg900\n", - " World\n", - " Gas\n", - " GW\n", - " 1096.6736\n", - " 1343.1077\n", - " 1603.1686\n", - " 1852.6485\n", - " 1761.8057\n", - " 1584.4033\n", - " 1423.5393\n", - " 1289.4777\n", + " REMIND-MAgPIE 2.1-4.3\n", + " DeepElec_SSP2_HighRE_Budg900\n", + " World\n", + " GWh/yr\n", + " Secondary Energy|Electricity|Biomass\n", + " 2.352778e+05\n", + " 3.407222e+05\n", + " 4.669444e+05\n", + " 5.942778e+05\n", + " 7.219167e+05\n", + " 8.496389e+05\n", + " 9.855833e+05\n", + " 1.173833e+06\n", + " 1.359639e+06\n", + " 1.471139e+06\n", + " 1.513056e+06\n", + " 1.544750e+06\n", + " 1.564583e+06\n", + " 1.557556e+06\n", + " 1.543056e+06\n", + " 1.574306e+06\n", " \n", " \n", - " Nuclear\n", - " GW\n", - " 408.3700\n", - " 404.3094\n", - " 379.1375\n", - " 345.2239\n", - " 310.1729\n", - " 275.5920\n", - " 234.7028\n", - " 214.4376\n", + " Secondary Energy|Electricity|Coal\n", + " 7.419778e+06\n", + " 8.252667e+06\n", + " 9.246333e+06\n", + " 1.075747e+07\n", + " 6.045972e+06\n", + " 8.733611e+05\n", + " 6.444444e+03\n", + " 5.027778e+03\n", + " 3.694444e+03\n", + " 2.416667e+03\n", + " 1.555556e+03\n", + " 9.444444e+02\n", + " 4.444444e+02\n", + " 5.277778e+02\n", + " 5.555556e+02\n", + " 5.000000e+02\n", " \n", - " \n", - "\n", - "" - ], - "text/plain": [ - " 2005 \\\n", - "model scenario region variable unit \n", - "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1096.6736 \n", - " Nuclear GW 408.3700 \n", + " \n", + " Secondary Energy|Electricity|Gas\n", + " 3.816028e+06\n", + " 5.050500e+06\n", + " 5.968778e+06\n", + " 7.060083e+06\n", + " 6.469250e+06\n", + " 4.551528e+06\n", + " 2.907250e+06\n", + " 2.602278e+06\n", + " 1.752056e+06\n", + " 1.218361e+06\n", + " 6.538889e+05\n", + " 5.281944e+05\n", + " 2.423333e+05\n", + " 1.685833e+05\n", + " 4.166667e+04\n", + " 3.888889e+02\n", + " \n", + " \n", + " Secondary Energy|Electricity|Geothermal\n", + " 6.041667e+04\n", + " 9.677778e+04\n", + " 1.799167e+05\n", + " 3.474167e+05\n", + " 5.260278e+05\n", + " 6.213611e+05\n", + " 6.291111e+05\n", + " 6.177778e+05\n", + " 5.893056e+05\n", + " 5.187222e+05\n", + " 4.333611e+05\n", + " 3.603889e+05\n", + " 2.924722e+05\n", + " 2.492778e+05\n", + " 2.059167e+05\n", + " 1.937500e+05\n", + " \n", + " \n", + " Secondary Energy|Electricity|Hydro\n", + " 2.976944e+06\n", + " 3.557028e+06\n", + " 4.386694e+06\n", + " 5.302056e+06\n", + " 5.813167e+06\n", + " 6.173778e+06\n", + " 6.423194e+06\n", + " 6.604167e+06\n", + " 6.717417e+06\n", + " 6.770250e+06\n", + " 6.782028e+06\n", + " 6.775917e+06\n", + " 6.722667e+06\n", + " 6.655833e+06\n", + " 6.631806e+06\n", + " 6.533167e+06\n", + " \n", + " \n", + " Secondary Energy|Electricity|Nuclear\n", + " 2.861861e+06\n", + " 2.642750e+06\n", + " 2.448556e+06\n", + " 2.214167e+06\n", + " 2.055667e+06\n", + " 1.876222e+06\n", + " 1.617278e+06\n", + " 1.502778e+06\n", + " 1.304083e+06\n", + " 1.098694e+06\n", + " 8.686111e+05\n", + " 6.451944e+05\n", + " 2.717500e+05\n", + " 5.925000e+04\n", + " 3.916667e+03\n", + " 1.388889e+02\n", + " \n", + " \n", + " Secondary Energy|Electricity|Oil\n", + " 1.088861e+06\n", + " 1.016972e+06\n", + " 8.353889e+05\n", + " 6.174167e+05\n", + " 4.317778e+05\n", + " 1.960556e+05\n", + " 6.919444e+04\n", + " 4.797222e+04\n", + " 3.175000e+04\n", + " 1.169444e+04\n", + " 6.722222e+03\n", + " 2.777778e+01\n", + " 2.777778e+01\n", + " 2.777778e+01\n", + " 2.777778e+01\n", + " 2.777778e+01\n", + " \n", + " \n", + " Secondary Energy|Electricity|Other\n", + " 0.000000e+00\n", + " 0.000000e+00\n", + " 4.444444e+02\n", + " 2.166667e+03\n", + " 8.913889e+04\n", + " 4.938889e+05\n", + " 1.204583e+06\n", + " 1.985111e+06\n", + " 2.790639e+06\n", + " 3.488083e+06\n", + " 4.020806e+06\n", + " 4.434639e+06\n", + " 5.261472e+06\n", + " 6.099583e+06\n", + " 6.740833e+06\n", + " 7.547194e+06\n", + " \n", + " \n", + " Secondary Energy|Electricity|Solar\n", + " 4.083333e+03\n", + " 8.325000e+04\n", + " 2.616111e+05\n", + " 9.263611e+05\n", + " 5.500722e+06\n", + " 1.566242e+07\n", + " 2.575731e+07\n", + " 3.404253e+07\n", + " 4.088394e+07\n", + " 4.723214e+07\n", + " 5.327061e+07\n", + " 5.819236e+07\n", + " 6.826367e+07\n", + " 7.754106e+07\n", + " 8.359456e+07\n", + " 9.270394e+07\n", + " \n", + " \n", + " Secondary Energy|Electricity|Wind\n", + " 1.091389e+05\n", + " 5.946944e+05\n", + " 8.323889e+05\n", + " 1.482806e+06\n", + " 3.695778e+06\n", + " 8.141417e+06\n", + " 1.318953e+07\n", + " 1.727517e+07\n", + " 2.112919e+07\n", + " 2.367689e+07\n", + " 2.569986e+07\n", + " 2.771675e+07\n", + " 3.280081e+07\n", + " 3.570786e+07\n", + " 3.840797e+07\n", + " 4.123369e+07\n", + " \n", + " \n", + "\n", + "" + ], + "text/plain": [ + " 2005 \\\n", + "model scenario region unit variable \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Secondary Energy|Electricity|Biomass 2.352778e+05 \n", + " Secondary Energy|Electricity|Coal 7.419778e+06 \n", + " Secondary Energy|Electricity|Gas 3.816028e+06 \n", + " Secondary Energy|Electricity|Geothermal 6.041667e+04 \n", + " Secondary Energy|Electricity|Hydro 2.976944e+06 \n", + " Secondary Energy|Electricity|Nuclear 2.861861e+06 \n", + " Secondary Energy|Electricity|Oil 1.088861e+06 \n", + " Secondary Energy|Electricity|Other 0.000000e+00 \n", + " Secondary Energy|Electricity|Solar 4.083333e+03 \n", + " Secondary Energy|Electricity|Wind 1.091389e+05 \n", "\n", - " 2010 \\\n", - "model scenario region variable unit \n", - "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1343.1077 \n", - " Nuclear GW 404.3094 \n", + " 2010 \\\n", + "model scenario region unit variable \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Secondary Energy|Electricity|Biomass 3.407222e+05 \n", + " Secondary Energy|Electricity|Coal 8.252667e+06 \n", + " Secondary Energy|Electricity|Gas 5.050500e+06 \n", + " Secondary Energy|Electricity|Geothermal 9.677778e+04 \n", + " Secondary Energy|Electricity|Hydro 3.557028e+06 \n", + " Secondary Energy|Electricity|Nuclear 2.642750e+06 \n", + " Secondary Energy|Electricity|Oil 1.016972e+06 \n", + " Secondary Energy|Electricity|Other 0.000000e+00 \n", + " Secondary Energy|Electricity|Solar 8.325000e+04 \n", + " Secondary Energy|Electricity|Wind 5.946944e+05 \n", "\n", - " 2015 \\\n", - "model scenario region variable unit \n", - "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1603.1686 \n", - " Nuclear GW 379.1375 \n", + " 2015 \\\n", + "model scenario region unit variable \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Secondary Energy|Electricity|Biomass 4.669444e+05 \n", + " Secondary Energy|Electricity|Coal 9.246333e+06 \n", + " Secondary Energy|Electricity|Gas 5.968778e+06 \n", + " Secondary Energy|Electricity|Geothermal 1.799167e+05 \n", + " Secondary Energy|Electricity|Hydro 4.386694e+06 \n", + " Secondary Energy|Electricity|Nuclear 2.448556e+06 \n", + " Secondary Energy|Electricity|Oil 8.353889e+05 \n", + " Secondary Energy|Electricity|Other 4.444444e+02 \n", + " Secondary Energy|Electricity|Solar 2.616111e+05 \n", + " Secondary Energy|Electricity|Wind 8.323889e+05 \n", "\n", - " 2020 \\\n", - "model scenario region variable unit \n", - "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1852.6485 \n", - " Nuclear GW 345.2239 \n", + " 2020 \\\n", + "model scenario region unit variable \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Secondary Energy|Electricity|Biomass 5.942778e+05 \n", + " Secondary Energy|Electricity|Coal 1.075747e+07 \n", + " Secondary Energy|Electricity|Gas 7.060083e+06 \n", + " Secondary Energy|Electricity|Geothermal 3.474167e+05 \n", + " Secondary Energy|Electricity|Hydro 5.302056e+06 \n", + " Secondary Energy|Electricity|Nuclear 2.214167e+06 \n", + " Secondary Energy|Electricity|Oil 6.174167e+05 \n", + " Secondary Energy|Electricity|Other 2.166667e+03 \n", + " Secondary Energy|Electricity|Solar 9.263611e+05 \n", + " Secondary Energy|Electricity|Wind 1.482806e+06 \n", "\n", - " 2025 \\\n", - "model scenario region variable unit \n", - "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1761.8057 \n", - " Nuclear GW 310.1729 \n", + " 2025 \\\n", + "model scenario region unit variable \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Secondary Energy|Electricity|Biomass 7.219167e+05 \n", + " Secondary Energy|Electricity|Coal 6.045972e+06 \n", + " Secondary Energy|Electricity|Gas 6.469250e+06 \n", + " Secondary Energy|Electricity|Geothermal 5.260278e+05 \n", + " Secondary Energy|Electricity|Hydro 5.813167e+06 \n", + " Secondary Energy|Electricity|Nuclear 2.055667e+06 \n", + " Secondary Energy|Electricity|Oil 4.317778e+05 \n", + " Secondary Energy|Electricity|Other 8.913889e+04 \n", + " Secondary Energy|Electricity|Solar 5.500722e+06 \n", + " Secondary Energy|Electricity|Wind 3.695778e+06 \n", "\n", - " 2030 \\\n", - "model scenario region variable unit \n", - "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1584.4033 \n", - " Nuclear GW 275.5920 \n", + " 2030 \\\n", + "model scenario region unit variable \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Secondary Energy|Electricity|Biomass 8.496389e+05 \n", + " Secondary Energy|Electricity|Coal 8.733611e+05 \n", + " Secondary Energy|Electricity|Gas 4.551528e+06 \n", + " Secondary Energy|Electricity|Geothermal 6.213611e+05 \n", + " Secondary Energy|Electricity|Hydro 6.173778e+06 \n", + " Secondary Energy|Electricity|Nuclear 1.876222e+06 \n", + " Secondary Energy|Electricity|Oil 1.960556e+05 \n", + " Secondary Energy|Electricity|Other 4.938889e+05 \n", + " Secondary Energy|Electricity|Solar 1.566242e+07 \n", + " Secondary Energy|Electricity|Wind 8.141417e+06 \n", "\n", - " 2035 \\\n", - "model scenario region variable unit \n", - "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1423.5393 \n", - " Nuclear GW 234.7028 \n", + " 2035 \\\n", + "model scenario region unit variable \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Secondary Energy|Electricity|Biomass 9.855833e+05 \n", + " Secondary Energy|Electricity|Coal 6.444444e+03 \n", + " Secondary Energy|Electricity|Gas 2.907250e+06 \n", + " Secondary Energy|Electricity|Geothermal 6.291111e+05 \n", + " Secondary Energy|Electricity|Hydro 6.423194e+06 \n", + " Secondary Energy|Electricity|Nuclear 1.617278e+06 \n", + " Secondary Energy|Electricity|Oil 6.919444e+04 \n", + " Secondary Energy|Electricity|Other 1.204583e+06 \n", + " Secondary Energy|Electricity|Solar 2.575731e+07 \n", + " Secondary Energy|Electricity|Wind 1.318953e+07 \n", "\n", - " 2040 \n", - "model scenario region variable unit \n", - "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1289.4777 \n", - " Nuclear GW 214.4376 " + " 2040 \\\n", + "model scenario region unit variable \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Secondary Energy|Electricity|Biomass 1.173833e+06 \n", + " Secondary Energy|Electricity|Coal 5.027778e+03 \n", + " Secondary Energy|Electricity|Gas 2.602278e+06 \n", + " Secondary Energy|Electricity|Geothermal 6.177778e+05 \n", + " Secondary Energy|Electricity|Hydro 6.604167e+06 \n", + " Secondary Energy|Electricity|Nuclear 1.502778e+06 \n", + " Secondary Energy|Electricity|Oil 4.797222e+04 \n", + " Secondary Energy|Electricity|Other 1.985111e+06 \n", + " Secondary Energy|Electricity|Solar 3.404253e+07 \n", + " Secondary Energy|Electricity|Wind 1.727517e+07 \n", + "\n", + " 2045 \\\n", + "model scenario region unit variable \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Secondary Energy|Electricity|Biomass 1.359639e+06 \n", + " Secondary Energy|Electricity|Coal 3.694444e+03 \n", + " Secondary Energy|Electricity|Gas 1.752056e+06 \n", + " Secondary Energy|Electricity|Geothermal 5.893056e+05 \n", + " Secondary Energy|Electricity|Hydro 6.717417e+06 \n", + " Secondary Energy|Electricity|Nuclear 1.304083e+06 \n", + " Secondary Energy|Electricity|Oil 3.175000e+04 \n", + " Secondary Energy|Electricity|Other 2.790639e+06 \n", + " Secondary Energy|Electricity|Solar 4.088394e+07 \n", + " Secondary Energy|Electricity|Wind 2.112919e+07 \n", + "\n", + " 2050 \\\n", + "model scenario region unit variable \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Secondary Energy|Electricity|Biomass 1.471139e+06 \n", + " Secondary Energy|Electricity|Coal 2.416667e+03 \n", + " Secondary Energy|Electricity|Gas 1.218361e+06 \n", + " Secondary Energy|Electricity|Geothermal 5.187222e+05 \n", + " Secondary Energy|Electricity|Hydro 6.770250e+06 \n", + " Secondary Energy|Electricity|Nuclear 1.098694e+06 \n", + " Secondary Energy|Electricity|Oil 1.169444e+04 \n", + " Secondary Energy|Electricity|Other 3.488083e+06 \n", + " Secondary Energy|Electricity|Solar 4.723214e+07 \n", + " Secondary Energy|Electricity|Wind 2.367689e+07 \n", + "\n", + " 2055 \\\n", + "model scenario region unit variable \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Secondary Energy|Electricity|Biomass 1.513056e+06 \n", + " Secondary Energy|Electricity|Coal 1.555556e+03 \n", + " Secondary Energy|Electricity|Gas 6.538889e+05 \n", + " Secondary Energy|Electricity|Geothermal 4.333611e+05 \n", + " Secondary Energy|Electricity|Hydro 6.782028e+06 \n", + " Secondary Energy|Electricity|Nuclear 8.686111e+05 \n", + " Secondary Energy|Electricity|Oil 6.722222e+03 \n", + " Secondary Energy|Electricity|Other 4.020806e+06 \n", + " Secondary Energy|Electricity|Solar 5.327061e+07 \n", + " Secondary Energy|Electricity|Wind 2.569986e+07 \n", + "\n", + " 2060 \\\n", + "model scenario region unit variable \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Secondary Energy|Electricity|Biomass 1.544750e+06 \n", + " Secondary Energy|Electricity|Coal 9.444444e+02 \n", + " Secondary Energy|Electricity|Gas 5.281944e+05 \n", + " Secondary Energy|Electricity|Geothermal 3.603889e+05 \n", + " Secondary Energy|Electricity|Hydro 6.775917e+06 \n", + " Secondary Energy|Electricity|Nuclear 6.451944e+05 \n", + " Secondary Energy|Electricity|Oil 2.777778e+01 \n", + " Secondary Energy|Electricity|Other 4.434639e+06 \n", + " Secondary Energy|Electricity|Solar 5.819236e+07 \n", + " Secondary Energy|Electricity|Wind 2.771675e+07 \n", + "\n", + " 2070 \\\n", + "model scenario region unit variable \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Secondary Energy|Electricity|Biomass 1.564583e+06 \n", + " Secondary Energy|Electricity|Coal 4.444444e+02 \n", + " Secondary Energy|Electricity|Gas 2.423333e+05 \n", + " Secondary Energy|Electricity|Geothermal 2.924722e+05 \n", + " Secondary Energy|Electricity|Hydro 6.722667e+06 \n", + " Secondary Energy|Electricity|Nuclear 2.717500e+05 \n", + " Secondary Energy|Electricity|Oil 2.777778e+01 \n", + " Secondary Energy|Electricity|Other 5.261472e+06 \n", + " Secondary Energy|Electricity|Solar 6.826367e+07 \n", + " Secondary Energy|Electricity|Wind 3.280081e+07 \n", + "\n", + " 2080 \\\n", + "model scenario region unit variable \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Secondary Energy|Electricity|Biomass 1.557556e+06 \n", + " Secondary Energy|Electricity|Coal 5.277778e+02 \n", + " Secondary Energy|Electricity|Gas 1.685833e+05 \n", + " Secondary Energy|Electricity|Geothermal 2.492778e+05 \n", + " Secondary Energy|Electricity|Hydro 6.655833e+06 \n", + " Secondary Energy|Electricity|Nuclear 5.925000e+04 \n", + " Secondary Energy|Electricity|Oil 2.777778e+01 \n", + " Secondary Energy|Electricity|Other 6.099583e+06 \n", + " Secondary Energy|Electricity|Solar 7.754106e+07 \n", + " Secondary Energy|Electricity|Wind 3.570786e+07 \n", + "\n", + " 2090 \\\n", + "model scenario region unit variable \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Secondary Energy|Electricity|Biomass 1.543056e+06 \n", + " Secondary Energy|Electricity|Coal 5.555556e+02 \n", + " Secondary Energy|Electricity|Gas 4.166667e+04 \n", + " Secondary Energy|Electricity|Geothermal 2.059167e+05 \n", + " Secondary Energy|Electricity|Hydro 6.631806e+06 \n", + " Secondary Energy|Electricity|Nuclear 3.916667e+03 \n", + " Secondary Energy|Electricity|Oil 2.777778e+01 \n", + " Secondary Energy|Electricity|Other 6.740833e+06 \n", + " Secondary Energy|Electricity|Solar 8.359456e+07 \n", + " Secondary Energy|Electricity|Wind 3.840797e+07 \n", + "\n", + " 2100 \n", + "model scenario region unit variable \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GWh/yr Secondary Energy|Electricity|Biomass 1.574306e+06 \n", + " Secondary Energy|Electricity|Coal 5.000000e+02 \n", + " Secondary Energy|Electricity|Gas 3.888889e+02 \n", + " Secondary Energy|Electricity|Geothermal 1.937500e+05 \n", + " Secondary Energy|Electricity|Hydro 6.533167e+06 \n", + " Secondary Energy|Electricity|Nuclear 1.388889e+02 \n", + " Secondary Energy|Electricity|Oil 2.777778e+01 \n", + " Secondary Energy|Electricity|Other 7.547194e+06 \n", + " Secondary Energy|Electricity|Solar 9.270394e+07 \n", + " Secondary Energy|Electricity|Wind 4.123369e+07 " ] }, - "execution_count": 10, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "high_capacity_fossil = capacity.loc[\n", - " isin(variable=[\"Coal\", \"Gas\", \"Nuclear\"], unit=\"GW\") & (capacity[2030] > 250),\n", - " :2041,\n", - "]\n", - "high_capacity_fossil" + "formatlevel(splitdf, variable=\"Secondary Energy|{carrier}|{fuel}\", drop=True)\n", + "# or: df.idx.format(variable=\"Secondary Energy|{carrier}|{fuel}\")" ] }, { "attachments": {}, "cell_type": "markdown", - "id": "50fe70b7", + "id": "e78ace7d", "metadata": {}, "source": [ - "The simple fact that this is an operation on `[]`, means that we can also use it to modify values in-place:" + "With `generation` and `capacity` conveniently split into separate variables, we can calculate capacity factors (ratios of generation and capacity) directly, as long as we take care of removing the conflicting `unit` level. Similarly to `ismatch`, `isin` can be provided as an argument to `.loc[]` to select on named index levels with the difference that only exact matches are considered." ] }, { "cell_type": "code", - "execution_count": 11, - "id": "71d7aa7d", + "execution_count": 10, + "id": "9b4be2b0", "metadata": {}, "outputs": [ { @@ -1385,25 +2060,17 @@ " \n", " \n", " \n", - " \n", - " 2005\n", - " 2010\n", - " 2015\n", - " 2020\n", - " 2025\n", " 2030\n", " 2035\n", " 2040\n", + " 2045\n", + " 2050\n", " \n", " \n", " model\n", " scenario\n", " region\n", " variable\n", - " unit\n", - " \n", - " \n", - " \n", " \n", " \n", " \n", @@ -1413,435 +2080,1660 @@ " \n", " \n", " \n", - " REMIND-MAgPIE 2.1-4.3\n", - " DeepElec_SSP2_HighRE_Budg900\n", - " World\n", - " Gas\n", - " GW\n", - " 1096.6736\n", - " 1343.1077\n", - " 1603.1686\n", - " 1852.6485\n", - " 1761.8057\n", - " 1000.000\n", - " 1000.0000\n", - " 1000.0000\n", + " REMIND-MAgPIE 2.1-4.3\n", + " DeepElec_SSP2_HighRE_Budg900\n", + " World\n", + " Geothermal\n", + " 0.849984\n", + " 0.850005\n", + " 0.849989\n", + " 0.849986\n", + " 0.850021\n", " \n", " \n", - " Nuclear\n", - " GW\n", - " 408.3700\n", - " 404.3094\n", - " 379.1375\n", - " 345.2239\n", - " 310.1729\n", - " 275.592\n", - " 234.7028\n", - " 214.4376\n", + " Hydro\n", + " 0.394597\n", + " 0.394474\n", + " 0.396240\n", + " 0.398217\n", + " 0.399716\n", " \n", - " \n", - "\n", - "" - ], + " \n", + " Solar\n", + " 0.161156\n", + " 0.154237\n", + " 0.151994\n", + " 0.150698\n", + " 0.149422\n", + " \n", + " \n", + " Wind\n", + " 0.263820\n", + " 0.275781\n", + " 0.289083\n", + " 0.302018\n", + " 0.298163\n", + " \n", + " \n", + "\n", + "" + ], "text/plain": [ - " 2005 \\\n", - "model scenario region variable unit \n", - "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1096.6736 \n", - " Nuclear GW 408.3700 \n", - "\n", - " 2010 \\\n", - "model scenario region variable unit \n", - "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1343.1077 \n", - " Nuclear GW 404.3094 \n", - "\n", - " 2015 \\\n", - "model scenario region variable unit \n", - "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1603.1686 \n", - " Nuclear GW 379.1375 \n", - "\n", - " 2020 \\\n", - "model scenario region variable unit \n", - "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1852.6485 \n", - " Nuclear GW 345.2239 \n", + " 2030 \\\n", + "model scenario region variable \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Geothermal 0.849984 \n", + " Hydro 0.394597 \n", + " Solar 0.161156 \n", + " Wind 0.263820 \n", "\n", - " 2025 \\\n", - "model scenario region variable unit \n", - "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1761.8057 \n", - " Nuclear GW 310.1729 \n", + " 2035 \\\n", + "model scenario region variable \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Geothermal 0.850005 \n", + " Hydro 0.394474 \n", + " Solar 0.154237 \n", + " Wind 0.275781 \n", "\n", - " 2030 \\\n", - "model scenario region variable unit \n", - "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1000.000 \n", - " Nuclear GW 275.592 \n", + " 2040 \\\n", + "model scenario region variable \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Geothermal 0.849989 \n", + " Hydro 0.396240 \n", + " Solar 0.151994 \n", + " Wind 0.289083 \n", "\n", - " 2035 \\\n", - "model scenario region variable unit \n", - "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1000.0000 \n", - " Nuclear GW 234.7028 \n", + " 2045 \\\n", + "model scenario region variable \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Geothermal 0.849986 \n", + " Hydro 0.398217 \n", + " Solar 0.150698 \n", + " Wind 0.302018 \n", "\n", - " 2040 \n", - "model scenario region variable unit \n", - "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1000.0000 \n", - " Nuclear GW 214.4376 " + " 2050 \n", + "model scenario region variable \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Geothermal 0.850021 \n", + " Hydro 0.399716 \n", + " Solar 0.149422 \n", + " Wind 0.298163 " ] }, - "execution_count": 11, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "high_capacity_fossil.loc[isin(variable=\"Gas\"), 2030:] = 1000.0\n", - "high_capacity_fossil" + "capacity_factor = generation.droplevel(\"unit\") / 8760 / capacity.droplevel(\"unit\")\n", + "capacity_factor.loc[isin(variable=[\"Solar\", \"Wind\", \"Hydro\", \"Geothermal\"]), 2030:2051]" ] }, { "attachments": {}, "cell_type": "markdown", - "id": "25dd3f6e", - "metadata": {}, - "source": [ - "Most methods in `pandas_indexing` do not care whether they are run on an index, a series or a dataframe, but will transiently take care of handing them down to the appropriate level:" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "e0ff32d5", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "model scenario region variable unit year\n", - "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Coal GW 2030 182.0149\n", - " 2040 1.0729\n", - " 2050 0.5029\n", - " 2060 0.1836\n", - " Gas GW 2030 1584.4033\n", - " 2040 1289.4777\n", - " 2050 562.8482\n", - " 2060 349.9091\n", - " Nuclear GW 2030 275.5920\n", - " 2040 214.4376\n", - " 2050 156.7766\n", - " 2060 92.0667\n", - "dtype: float64" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "fossil_series = (\n", - " capacity.loc[isin(variable=[\"Coal\", \"Gas\", \"Nuclear\"]), [2030, 2040, 2050, 2060]]\n", - " .rename_axis(columns=\"year\")\n", - " .stack()\n", - ")\n", - "fossil_series" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "6d4686e6", + "id": "b4d501a3", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "model scenario region variable unit year\n", - "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Coal GW 2030 182.0149\n", - " 2050 0.5029\n", - " Gas GW 2030 1584.4033\n", - " 2050 562.8482\n", - " Nuclear GW 2030 275.5920\n", - " 2050 156.7766\n", - "dtype: float64" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ - "fossil_series.loc[isin(year=[2030, 2050])]" + "Under the hood `isin` and `ismatch` generate `Selector` objects. They can be composed into complex queries intuitively, which are kept as a hierarchical structure of objects." ] }, { "cell_type": "code", - "execution_count": 14, - "id": "8d4e91fb", + "execution_count": 11, + "id": "4197be7d", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "array([False, False, False, False, False, False, False, False, True,\n", - " True, True, True])" + "And(a=Isin(filters={'variable': ['Coal', 'Gas', 'Nuclear'], 'unit': 'GW'}), b=Not(a=Ismatch(filters={'variable': 'S*'}, regex=False)))" ] }, - "execution_count": 14, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "isin(fossil_series.index, variable=\"Nuclear\")" + "query = isin(variable=[\"Coal\", \"Gas\", \"Nuclear\"], unit=\"GW\") & ~ismatch(variable=\"S*\")\n", + "query" ] }, { "attachments": {}, "cell_type": "markdown", - "id": "36c41351", - "metadata": {}, - "source": [ - "## Selecting based on a multi-index\n", - "\n", - "If we need pairs of data like `Coal` in 2030 and `Gas` in 2040 and `Nuclear` in 2040 and 2050, based on a given multiindex, then this defines right-oriented `semijoin` like:" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "cde4ff9f", + "id": "f137127f", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "MultiIndex([( 'Coal', 2030),\n", - " ( 'Gas', 2035),\n", - " ('Nuclear', 2040),\n", - " ('Nuclear', 2050)],\n", - " names=['variable', 'year'])" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ - "idx = pd.MultiIndex.from_tuples(\n", - " [(\"Coal\", 2030), (\"Gas\", 2035), (\"Nuclear\", 2040), (\"Nuclear\", 2050)],\n", - " names=[\"variable\", \"year\"],\n", - ")\n", - "idx" + "For evaluating such a query one needs to pass in a data object to produce a boolean mask. Since pandas `.loc` indexer does exactly that, these queries work as expected.\n" ] }, { "cell_type": "code", - "execution_count": 16, - "id": "dc294f2f", + "execution_count": 12, + "id": "18401aa0", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "variable year model scenario region unit\n", - "Coal 2030 REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GW 182.0149\n", - "Gas 2035 NaN NaN NaN NaN NaN\n", - "Nuclear 2040 REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GW 214.4376\n", - " 2050 REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GW 156.7766\n", - "dtype: float64" + "model scenario region variable unit\n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Biomass GW False\n", + " Coal GW True\n", + " Gas GW True\n", + " Geothermal GW False\n", + " Hydro GW False\n", + " Nuclear GW True\n", + " Oil GW False\n", + " Other GW False\n", + " Solar GW False\n", + " Wind GW False\n", + "dtype: bool" ] }, - "execution_count": 16, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "from pandas_indexing import semijoin\n", - "\n", - "semijoin(fossil_series, idx, how=\"right\")" + "query(capacity)" ] }, { "attachments": {}, "cell_type": "markdown", - "id": "cbaa73b0", + "id": "7e1d4f00", "metadata": {}, "source": [ - "Since the `(\"Gas\", 2035)` is not part of the original `fossil_series` it shows up as `NaN`s here, an inner join will skip it silently:" + "
\n", + "\n", + "WARNING\n", + "\n", + "It is currently impossible to use a pandas boolean series **in front of** a selector; ie.\n", + "\n", + "```python\n", + "(capacity[2030] > 250) & isin(variable=[\"Coal\", \"Gas\", \"Nuclear\"], unit=\"GW\")\n", + "```\n", + "will **fail**, it needs to be\n", + "```python\n", + "isin(variable=[\"Coal\", \"Gas\", \"Nuclear\"], unit=\"GW\") & (capacity[2030] > 250)\n", + "```\n", + "\n", + "
" ] }, { "cell_type": "code", - "execution_count": 17, - "id": "f82d6bf4", + "execution_count": 13, + "id": "8b421dba", "metadata": {}, "outputs": [ { "data": { - "text/plain": [ - "variable year model scenario region unit\n", - "Coal 2030 REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GW 182.0149\n", - "Nuclear 2040 REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GW 214.4376\n", - " 2050 REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GW 156.7766\n", - "dtype: float64" + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
20052010201520202025203020352040
modelscenarioregionvariableunit
REMIND-MAgPIE 2.1-4.3DeepElec_SSP2_HighRE_Budg900WorldGasGW1096.67361343.10771603.16861852.64851761.80571584.40331423.53931289.4777
NuclearGW408.3700404.3094379.1375345.2239310.1729275.5920234.7028214.4376
\n", + "
" + ], + "text/plain": [ + " 2005 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1096.6736 \n", + " Nuclear GW 408.3700 \n", + "\n", + " 2010 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1343.1077 \n", + " Nuclear GW 404.3094 \n", + "\n", + " 2015 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1603.1686 \n", + " Nuclear GW 379.1375 \n", + "\n", + " 2020 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1852.6485 \n", + " Nuclear GW 345.2239 \n", + "\n", + " 2025 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1761.8057 \n", + " Nuclear GW 310.1729 \n", + "\n", + " 2030 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1584.4033 \n", + " Nuclear GW 275.5920 \n", + "\n", + " 2035 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1423.5393 \n", + " Nuclear GW 234.7028 \n", + "\n", + " 2040 \n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1289.4777 \n", + " Nuclear GW 214.4376 " ] }, - "execution_count": 17, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "semijoin(fossil_series, idx, how=\"inner\")" + "high_capacity_fossil = capacity.loc[\n", + " isin(variable=[\"Coal\", \"Gas\", \"Nuclear\"], unit=\"GW\") & (capacity[2030] > 250),\n", + " :2041,\n", + "]\n", + "high_capacity_fossil" ] }, { "attachments": {}, "cell_type": "markdown", - "id": "f2e1e0e4", + "id": "50fe70b7", "metadata": {}, "source": [ - "## Projecting levels\n", - "\n", - "Often after selecting the right subsets, ie the interesting `model` or `scenario` it makes sense to consolidate the data to a given set of `levels`. That is what `projectlevel` is used for:" + "The simple fact that this is an operation on `[]`, means that we can also use it to modify values in-place:" ] }, { "cell_type": "code", - "execution_count": 18, - "id": "6d0000aa", + "execution_count": 14, + "id": "71d7aa7d", "metadata": {}, "outputs": [ { "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
20052010201520202025203020352040
modelscenarioregionvariableunit
REMIND-MAgPIE 2.1-4.3DeepElec_SSP2_HighRE_Budg900WorldGasGW1096.67361343.10771603.16861852.64851761.80571000.0001000.00001000.0000
NuclearGW408.3700404.3094379.1375345.2239310.1729275.592234.7028214.4376
\n", + "
" + ], "text/plain": [ - "variable year\n", - "Coal 2030 182.0149\n", - " 2040 1.0729\n", - " 2050 0.5029\n", - " 2060 0.1836\n", - "Gas 2030 1584.4033\n", - " 2040 1289.4777\n", - " 2050 562.8482\n", - " 2060 349.9091\n", - "Nuclear 2030 275.5920\n", - " 2040 214.4376\n", - " 2050 156.7766\n", - " 2060 92.0667\n", - "dtype: float64" + " 2005 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1096.6736 \n", + " Nuclear GW 408.3700 \n", + "\n", + " 2010 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1343.1077 \n", + " Nuclear GW 404.3094 \n", + "\n", + " 2015 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1603.1686 \n", + " Nuclear GW 379.1375 \n", + "\n", + " 2020 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1852.6485 \n", + " Nuclear GW 345.2239 \n", + "\n", + " 2025 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1761.8057 \n", + " Nuclear GW 310.1729 \n", + "\n", + " 2030 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1000.000 \n", + " Nuclear GW 275.592 \n", + "\n", + " 2035 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1000.0000 \n", + " Nuclear GW 234.7028 \n", + "\n", + " 2040 \n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Gas GW 1000.0000 \n", + " Nuclear GW 214.4376 " + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "high_capacity_fossil.loc[isin(variable=\"Gas\"), 2030:] = 1000.0\n", + "high_capacity_fossil" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "25dd3f6e", + "metadata": {}, + "source": [ + "Most methods in `pandas_indexing` do not care whether they are run on an index, a series or a dataframe, but will transiently take care of handing them down to the appropriate level:" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "e0ff32d5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "model scenario region variable unit year\n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Coal GW 2030 182.0149\n", + " 2040 1.0729\n", + " 2050 0.5029\n", + " 2060 0.1836\n", + " Gas GW 2030 1584.4033\n", + " 2040 1289.4777\n", + " 2050 562.8482\n", + " 2060 349.9091\n", + " Nuclear GW 2030 275.5920\n", + " 2040 214.4376\n", + " 2050 156.7766\n", + " 2060 92.0667\n", + "dtype: float64" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "fossil_series = (\n", + " capacity.loc[isin(variable=[\"Coal\", \"Gas\", \"Nuclear\"]), [2030, 2040, 2050, 2060]]\n", + " .rename_axis(columns=\"year\")\n", + " .stack()\n", + ")\n", + "fossil_series" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "6d4686e6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "model scenario region variable unit year\n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Coal GW 2030 182.0149\n", + " 2050 0.5029\n", + " Gas GW 2030 1584.4033\n", + " 2050 562.8482\n", + " Nuclear GW 2030 275.5920\n", + " 2050 156.7766\n", + "dtype: float64" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "fossil_series.loc[isin(year=[2030, 2050])]" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "8d4e91fb", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([False, False, False, False, False, False, False, False, True,\n", + " True, True, True])" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "isin(fossil_series.index, variable=\"Nuclear\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "36c41351", + "metadata": {}, + "source": [ + "# Selecting based on a multi-index\n", + "\n", + "If we need pairs of data like `Coal` in 2030 and `Gas` in 2040 and `Nuclear` in 2040 and 2050, based on a given multiindex, then this defines right-oriented `semijoin` like:" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "cde4ff9f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "MultiIndex([( 'Coal', 2030),\n", + " ( 'Gas', 2035),\n", + " ('Nuclear', 2040),\n", + " ('Nuclear', 2050)],\n", + " names=['variable', 'year'])" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "idx = pd.MultiIndex.from_tuples(\n", + " [(\"Coal\", 2030), (\"Gas\", 2035), (\"Nuclear\", 2040), (\"Nuclear\", 2050)],\n", + " names=[\"variable\", \"year\"],\n", + ")\n", + "idx" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "dc294f2f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "variable year model scenario region unit\n", + "Coal 2030 REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GW 182.0149\n", + "Gas 2035 NaN NaN NaN NaN NaN\n", + "Nuclear 2040 REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GW 214.4376\n", + " 2050 REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GW 156.7766\n", + "dtype: float64" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from pandas_indexing import semijoin\n", + "\n", + "semijoin(\n", + " fossil_series, idx, how=\"right\"\n", + ") # or: fossil_series.idx.semijoin(idx, how=\"right\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "cbaa73b0", + "metadata": {}, + "source": [ + "Since the `(\"Gas\", 2035)` is not part of the original `fossil_series` it shows up as `NaN`s here, an inner join will skip it silently:" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "f82d6bf4", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "variable year model scenario region unit\n", + "Coal 2030 REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GW 182.0149\n", + "Nuclear 2040 REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GW 214.4376\n", + " 2050 REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World GW 156.7766\n", + "dtype: float64" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "semijoin(fossil_series, idx, how=\"inner\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "f2e1e0e4", + "metadata": {}, + "source": [ + "# Projecting levels\n", + "\n", + "Often after selecting the right subsets, ie the interesting `model` or `scenario` it makes sense to consolidate the data to a given set of `levels`. That is what `projectlevel` is used for:" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "6d0000aa", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "variable year\n", + "Coal 2030 182.0149\n", + " 2040 1.0729\n", + " 2050 0.5029\n", + " 2060 0.1836\n", + "Gas 2030 1584.4033\n", + " 2040 1289.4777\n", + " 2050 562.8482\n", + " 2060 349.9091\n", + "Nuclear 2030 275.5920\n", + " 2040 214.4376\n", + " 2050 156.7766\n", + " 2060 92.0667\n", + "dtype: float64" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from pandas_indexing import projectlevel\n", + "\n", + "simple_fossil_series = projectlevel(fossil_series, [\"variable\", \"year\"])\n", + "# or: fossil_series.idx.project([\"variable\", \"year\"])\n", + "simple_fossil_series" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "715fe070", + "metadata": {}, + "source": [ + "`projectlevel` reduces the levels attached to a multiindex to the ones explicitly named. It is basically the complement to `droplevel` which removes the listed names" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "308fb920", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "model scenario \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 True\n", + " DeepElec_SSP2_HighRE_Budg900 True\n", + " DeepElec_SSP2_HighRE_Budg900 True\n", + " DeepElec_SSP2_HighRE_Budg900 True\n", + " DeepElec_SSP2_HighRE_Budg900 True\n", + " DeepElec_SSP2_HighRE_Budg900 True\n", + " DeepElec_SSP2_HighRE_Budg900 True\n", + " DeepElec_SSP2_HighRE_Budg900 True\n", + " DeepElec_SSP2_HighRE_Budg900 True\n", + " DeepElec_SSP2_HighRE_Budg900 True\n", + " DeepElec_SSP2_HighRE_Budg900 True\n", + " DeepElec_SSP2_HighRE_Budg900 True\n", + "dtype: bool" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "projectlevel(fossil_series, [\"model\", \"scenario\"]) == fossil_series.droplevel(\n", + " [\"variable\", \"unit\", \"region\", \"year\"]\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "6eb7b029", + "metadata": {}, + "source": [ + "# Assigning to levels\n", + "\n", + "`assignlevel` allows to modify individual values with helpful keyword arguments," + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "ca2a3c70", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
2005201020152020202520302035204020452050205520602070208020902100
modelscenarioregionvariableunit
REMIND-MAgPIE 2.1-4.3DeepElec_SSP2_HighRE_Budg900WorldUpdated|Capacity|Electricity|Biomassbla4.298430e+016.247170e+018.566980e+011.088834e+021.285653e+021.378452e+021.328316e+021.227382e+021.063772e+028.602650e+016.565480e+014.575910e+011.450470e+013.962800e+008.198000e-012.150000e-01
Updated|Capacity|Electricity|Coalbla1.238877e+031.525950e+031.876595e+032.075232e+031.225451e+031.820149e+021.353200e+001.072900e+007.763000e-015.029000e-013.111000e-011.836000e-017.830000e-029.180000e-029.650000e-028.660000e-02
Updated|Capacity|Electricity|Gasbla1.096674e+031.343108e+031.603169e+031.852649e+031.761806e+031.584403e+031.423539e+031.289478e+038.638716e+025.628482e+023.937209e+023.499091e+023.063883e+022.135970e+025.260450e+012.717000e-01
Updated|Capacity|Electricity|Geothermalbla8.112900e+001.299850e+012.416120e+014.665730e+017.064690e+018.345060e+018.448930e+018.296880e+017.914520e+016.966280e+015.820210e+014.840070e+013.927950e+013.347930e+012.765300e+012.601980e+01
Updated|Capacity|Electricity|Hydrobla8.065684e+029.633523e+021.239617e+031.500451e+031.669357e+031.786047e+031.858782e+031.902635e+031.925653e+031.933520e+031.931110e+031.922569e+031.893641e+031.860861e+031.846102e+031.805140e+03
Updated|Capacity|Electricity|Nuclearbla4.083700e+024.043094e+023.791375e+023.452239e+023.101729e+022.755920e+022.347028e+022.144376e+021.860849e+021.567766e+021.239472e+029.206670e+013.877600e+018.455000e+005.580000e-011.900000e-02
Updated|Capacity|Electricity|Oilbla5.112499e+024.630961e+023.709638e+022.817401e+021.932768e+029.023290e+013.159640e+012.190000e+011.450230e+015.341200e+003.064300e+001.200000e-021.200000e-021.200000e-021.210000e-021.260000e-02
Updated|Capacity|Electricity|Otherbla0.000000e+000.000000e+003.396000e-011.656200e+008.110980e+014.695231e+021.253078e+032.254017e+033.341699e+034.267072e+034.925403e+035.433398e+036.432168e+037.478326e+038.321384e+039.334424e+03
Updated|Capacity|Electricity|Solarbla2.682400e+004.961380e+012.198533e+026.509654e+023.785689e+031.109448e+041.906378e+042.556775e+043.097003e+043.608447e+044.110399e+044.527457e+045.443399e+046.305545e+046.884808e+047.743017e+04
Updated|Capacity|Electricity|Windbla3.226040e+011.811785e+024.124751e+026.682517e+021.616334e+033.522806e+035.459596e+036.821745e+037.986305e+039.064963e+039.934779e+031.082886e+041.320022e+041.459313e+041.601071e+041.736882e+04
Updated|Secondary Energy|Electricity|Biomassbla2.352778e+053.407222e+054.669444e+055.942778e+057.219167e+058.496389e+059.855833e+051.173833e+061.359639e+061.471139e+061.513056e+061.544750e+061.564583e+061.557556e+061.543056e+061.574306e+06
Updated|Secondary Energy|Electricity|Coalbla7.419778e+068.252667e+069.246333e+061.075747e+076.045972e+068.733611e+056.444444e+035.027778e+033.694444e+032.416667e+031.555556e+039.444444e+024.444444e+025.277778e+025.555556e+025.000000e+02
Updated|Secondary Energy|Electricity|Gasbla3.816028e+065.050500e+065.968778e+067.060083e+066.469250e+064.551528e+062.907250e+062.602278e+061.752056e+061.218361e+066.538889e+055.281944e+052.423333e+051.685833e+054.166667e+043.888889e+02
Updated|Secondary Energy|Electricity|Geothermalbla6.041667e+049.677778e+041.799167e+053.474167e+055.260278e+056.213611e+056.291111e+056.177778e+055.893056e+055.187222e+054.333611e+053.603889e+052.924722e+052.492778e+052.059167e+051.937500e+05
Updated|Secondary Energy|Electricity|Hydrobla2.976944e+063.557028e+064.386694e+065.302056e+065.813167e+066.173778e+066.423194e+066.604167e+066.717417e+066.770250e+066.782028e+066.775917e+066.722667e+066.655833e+066.631806e+066.533167e+06
Updated|Secondary Energy|Electricity|Nuclearbla2.861861e+062.642750e+062.448556e+062.214167e+062.055667e+061.876222e+061.617278e+061.502778e+061.304083e+061.098694e+068.686111e+056.451944e+052.717500e+055.925000e+043.916667e+031.388889e+02
Updated|Secondary Energy|Electricity|Oilbla1.088861e+061.016972e+068.353889e+056.174167e+054.317778e+051.960556e+056.919444e+044.797222e+043.175000e+041.169444e+046.722222e+032.777778e+012.777778e+012.777778e+012.777778e+012.777778e+01
Updated|Secondary Energy|Electricity|Otherbla0.000000e+000.000000e+004.444444e+022.166667e+038.913889e+044.938889e+051.204583e+061.985111e+062.790639e+063.488083e+064.020806e+064.434639e+065.261472e+066.099583e+066.740833e+067.547194e+06
Updated|Secondary Energy|Electricity|Solarbla4.083333e+038.325000e+042.616111e+059.263611e+055.500722e+061.566242e+072.575731e+073.404253e+074.088394e+074.723214e+075.327061e+075.819236e+076.826367e+077.754106e+078.359456e+079.270394e+07
Updated|Secondary Energy|Electricity|Windbla1.091389e+055.946944e+058.323889e+051.482806e+063.695778e+068.141417e+061.318953e+071.727517e+072.112919e+072.367689e+072.569986e+072.771675e+073.280081e+073.570786e+073.840797e+074.123369e+07
\n", + "
" + ], + "text/plain": [ + " 2005 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Updated|Capacity|Electricity|Biomass bla 4.298430e+01 \n", + " Updated|Capacity|Electricity|Coal bla 1.238877e+03 \n", + " Updated|Capacity|Electricity|Gas bla 1.096674e+03 \n", + " Updated|Capacity|Electricity|Geothermal bla 8.112900e+00 \n", + " Updated|Capacity|Electricity|Hydro bla 8.065684e+02 \n", + " Updated|Capacity|Electricity|Nuclear bla 4.083700e+02 \n", + " Updated|Capacity|Electricity|Oil bla 5.112499e+02 \n", + " Updated|Capacity|Electricity|Other bla 0.000000e+00 \n", + " Updated|Capacity|Electricity|Solar bla 2.682400e+00 \n", + " Updated|Capacity|Electricity|Wind bla 3.226040e+01 \n", + " Updated|Secondary Energy|Electricity|Biomass bla 2.352778e+05 \n", + " Updated|Secondary Energy|Electricity|Coal bla 7.419778e+06 \n", + " Updated|Secondary Energy|Electricity|Gas bla 3.816028e+06 \n", + " Updated|Secondary Energy|Electricity|Geothermal bla 6.041667e+04 \n", + " Updated|Secondary Energy|Electricity|Hydro bla 2.976944e+06 \n", + " Updated|Secondary Energy|Electricity|Nuclear bla 2.861861e+06 \n", + " Updated|Secondary Energy|Electricity|Oil bla 1.088861e+06 \n", + " Updated|Secondary Energy|Electricity|Other bla 0.000000e+00 \n", + " Updated|Secondary Energy|Electricity|Solar bla 4.083333e+03 \n", + " Updated|Secondary Energy|Electricity|Wind bla 1.091389e+05 \n", + "\n", + " 2010 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Updated|Capacity|Electricity|Biomass bla 6.247170e+01 \n", + " Updated|Capacity|Electricity|Coal bla 1.525950e+03 \n", + " Updated|Capacity|Electricity|Gas bla 1.343108e+03 \n", + " Updated|Capacity|Electricity|Geothermal bla 1.299850e+01 \n", + " Updated|Capacity|Electricity|Hydro bla 9.633523e+02 \n", + " Updated|Capacity|Electricity|Nuclear bla 4.043094e+02 \n", + " Updated|Capacity|Electricity|Oil bla 4.630961e+02 \n", + " Updated|Capacity|Electricity|Other bla 0.000000e+00 \n", + " Updated|Capacity|Electricity|Solar bla 4.961380e+01 \n", + " Updated|Capacity|Electricity|Wind bla 1.811785e+02 \n", + " Updated|Secondary Energy|Electricity|Biomass bla 3.407222e+05 \n", + " Updated|Secondary Energy|Electricity|Coal bla 8.252667e+06 \n", + " Updated|Secondary Energy|Electricity|Gas bla 5.050500e+06 \n", + " Updated|Secondary Energy|Electricity|Geothermal bla 9.677778e+04 \n", + " Updated|Secondary Energy|Electricity|Hydro bla 3.557028e+06 \n", + " Updated|Secondary Energy|Electricity|Nuclear bla 2.642750e+06 \n", + " Updated|Secondary Energy|Electricity|Oil bla 1.016972e+06 \n", + " Updated|Secondary Energy|Electricity|Other bla 0.000000e+00 \n", + " Updated|Secondary Energy|Electricity|Solar bla 8.325000e+04 \n", + " Updated|Secondary Energy|Electricity|Wind bla 5.946944e+05 \n", + "\n", + " 2015 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Updated|Capacity|Electricity|Biomass bla 8.566980e+01 \n", + " Updated|Capacity|Electricity|Coal bla 1.876595e+03 \n", + " Updated|Capacity|Electricity|Gas bla 1.603169e+03 \n", + " Updated|Capacity|Electricity|Geothermal bla 2.416120e+01 \n", + " Updated|Capacity|Electricity|Hydro bla 1.239617e+03 \n", + " Updated|Capacity|Electricity|Nuclear bla 3.791375e+02 \n", + " Updated|Capacity|Electricity|Oil bla 3.709638e+02 \n", + " Updated|Capacity|Electricity|Other bla 3.396000e-01 \n", + " Updated|Capacity|Electricity|Solar bla 2.198533e+02 \n", + " Updated|Capacity|Electricity|Wind bla 4.124751e+02 \n", + " Updated|Secondary Energy|Electricity|Biomass bla 4.669444e+05 \n", + " Updated|Secondary Energy|Electricity|Coal bla 9.246333e+06 \n", + " Updated|Secondary Energy|Electricity|Gas bla 5.968778e+06 \n", + " Updated|Secondary Energy|Electricity|Geothermal bla 1.799167e+05 \n", + " Updated|Secondary Energy|Electricity|Hydro bla 4.386694e+06 \n", + " Updated|Secondary Energy|Electricity|Nuclear bla 2.448556e+06 \n", + " Updated|Secondary Energy|Electricity|Oil bla 8.353889e+05 \n", + " Updated|Secondary Energy|Electricity|Other bla 4.444444e+02 \n", + " Updated|Secondary Energy|Electricity|Solar bla 2.616111e+05 \n", + " Updated|Secondary Energy|Electricity|Wind bla 8.323889e+05 \n", + "\n", + " 2020 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Updated|Capacity|Electricity|Biomass bla 1.088834e+02 \n", + " Updated|Capacity|Electricity|Coal bla 2.075232e+03 \n", + " Updated|Capacity|Electricity|Gas bla 1.852649e+03 \n", + " Updated|Capacity|Electricity|Geothermal bla 4.665730e+01 \n", + " Updated|Capacity|Electricity|Hydro bla 1.500451e+03 \n", + " Updated|Capacity|Electricity|Nuclear bla 3.452239e+02 \n", + " Updated|Capacity|Electricity|Oil bla 2.817401e+02 \n", + " Updated|Capacity|Electricity|Other bla 1.656200e+00 \n", + " Updated|Capacity|Electricity|Solar bla 6.509654e+02 \n", + " Updated|Capacity|Electricity|Wind bla 6.682517e+02 \n", + " Updated|Secondary Energy|Electricity|Biomass bla 5.942778e+05 \n", + " Updated|Secondary Energy|Electricity|Coal bla 1.075747e+07 \n", + " Updated|Secondary Energy|Electricity|Gas bla 7.060083e+06 \n", + " Updated|Secondary Energy|Electricity|Geothermal bla 3.474167e+05 \n", + " Updated|Secondary Energy|Electricity|Hydro bla 5.302056e+06 \n", + " Updated|Secondary Energy|Electricity|Nuclear bla 2.214167e+06 \n", + " Updated|Secondary Energy|Electricity|Oil bla 6.174167e+05 \n", + " Updated|Secondary Energy|Electricity|Other bla 2.166667e+03 \n", + " Updated|Secondary Energy|Electricity|Solar bla 9.263611e+05 \n", + " Updated|Secondary Energy|Electricity|Wind bla 1.482806e+06 \n", + "\n", + " 2025 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Updated|Capacity|Electricity|Biomass bla 1.285653e+02 \n", + " Updated|Capacity|Electricity|Coal bla 1.225451e+03 \n", + " Updated|Capacity|Electricity|Gas bla 1.761806e+03 \n", + " Updated|Capacity|Electricity|Geothermal bla 7.064690e+01 \n", + " Updated|Capacity|Electricity|Hydro bla 1.669357e+03 \n", + " Updated|Capacity|Electricity|Nuclear bla 3.101729e+02 \n", + " Updated|Capacity|Electricity|Oil bla 1.932768e+02 \n", + " Updated|Capacity|Electricity|Other bla 8.110980e+01 \n", + " Updated|Capacity|Electricity|Solar bla 3.785689e+03 \n", + " Updated|Capacity|Electricity|Wind bla 1.616334e+03 \n", + " Updated|Secondary Energy|Electricity|Biomass bla 7.219167e+05 \n", + " Updated|Secondary Energy|Electricity|Coal bla 6.045972e+06 \n", + " Updated|Secondary Energy|Electricity|Gas bla 6.469250e+06 \n", + " Updated|Secondary Energy|Electricity|Geothermal bla 5.260278e+05 \n", + " Updated|Secondary Energy|Electricity|Hydro bla 5.813167e+06 \n", + " Updated|Secondary Energy|Electricity|Nuclear bla 2.055667e+06 \n", + " Updated|Secondary Energy|Electricity|Oil bla 4.317778e+05 \n", + " Updated|Secondary Energy|Electricity|Other bla 8.913889e+04 \n", + " Updated|Secondary Energy|Electricity|Solar bla 5.500722e+06 \n", + " Updated|Secondary Energy|Electricity|Wind bla 3.695778e+06 \n", + "\n", + " 2030 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Updated|Capacity|Electricity|Biomass bla 1.378452e+02 \n", + " Updated|Capacity|Electricity|Coal bla 1.820149e+02 \n", + " Updated|Capacity|Electricity|Gas bla 1.584403e+03 \n", + " Updated|Capacity|Electricity|Geothermal bla 8.345060e+01 \n", + " Updated|Capacity|Electricity|Hydro bla 1.786047e+03 \n", + " Updated|Capacity|Electricity|Nuclear bla 2.755920e+02 \n", + " Updated|Capacity|Electricity|Oil bla 9.023290e+01 \n", + " Updated|Capacity|Electricity|Other bla 4.695231e+02 \n", + " Updated|Capacity|Electricity|Solar bla 1.109448e+04 \n", + " Updated|Capacity|Electricity|Wind bla 3.522806e+03 \n", + " Updated|Secondary Energy|Electricity|Biomass bla 8.496389e+05 \n", + " Updated|Secondary Energy|Electricity|Coal bla 8.733611e+05 \n", + " Updated|Secondary Energy|Electricity|Gas bla 4.551528e+06 \n", + " Updated|Secondary Energy|Electricity|Geothermal bla 6.213611e+05 \n", + " Updated|Secondary Energy|Electricity|Hydro bla 6.173778e+06 \n", + " Updated|Secondary Energy|Electricity|Nuclear bla 1.876222e+06 \n", + " Updated|Secondary Energy|Electricity|Oil bla 1.960556e+05 \n", + " Updated|Secondary Energy|Electricity|Other bla 4.938889e+05 \n", + " Updated|Secondary Energy|Electricity|Solar bla 1.566242e+07 \n", + " Updated|Secondary Energy|Electricity|Wind bla 8.141417e+06 \n", + "\n", + " 2035 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Updated|Capacity|Electricity|Biomass bla 1.328316e+02 \n", + " Updated|Capacity|Electricity|Coal bla 1.353200e+00 \n", + " Updated|Capacity|Electricity|Gas bla 1.423539e+03 \n", + " Updated|Capacity|Electricity|Geothermal bla 8.448930e+01 \n", + " Updated|Capacity|Electricity|Hydro bla 1.858782e+03 \n", + " Updated|Capacity|Electricity|Nuclear bla 2.347028e+02 \n", + " Updated|Capacity|Electricity|Oil bla 3.159640e+01 \n", + " Updated|Capacity|Electricity|Other bla 1.253078e+03 \n", + " Updated|Capacity|Electricity|Solar bla 1.906378e+04 \n", + " Updated|Capacity|Electricity|Wind bla 5.459596e+03 \n", + " Updated|Secondary Energy|Electricity|Biomass bla 9.855833e+05 \n", + " Updated|Secondary Energy|Electricity|Coal bla 6.444444e+03 \n", + " Updated|Secondary Energy|Electricity|Gas bla 2.907250e+06 \n", + " Updated|Secondary Energy|Electricity|Geothermal bla 6.291111e+05 \n", + " Updated|Secondary Energy|Electricity|Hydro bla 6.423194e+06 \n", + " Updated|Secondary Energy|Electricity|Nuclear bla 1.617278e+06 \n", + " Updated|Secondary Energy|Electricity|Oil bla 6.919444e+04 \n", + " Updated|Secondary Energy|Electricity|Other bla 1.204583e+06 \n", + " Updated|Secondary Energy|Electricity|Solar bla 2.575731e+07 \n", + " Updated|Secondary Energy|Electricity|Wind bla 1.318953e+07 \n", + "\n", + " 2040 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Updated|Capacity|Electricity|Biomass bla 1.227382e+02 \n", + " Updated|Capacity|Electricity|Coal bla 1.072900e+00 \n", + " Updated|Capacity|Electricity|Gas bla 1.289478e+03 \n", + " Updated|Capacity|Electricity|Geothermal bla 8.296880e+01 \n", + " Updated|Capacity|Electricity|Hydro bla 1.902635e+03 \n", + " Updated|Capacity|Electricity|Nuclear bla 2.144376e+02 \n", + " Updated|Capacity|Electricity|Oil bla 2.190000e+01 \n", + " Updated|Capacity|Electricity|Other bla 2.254017e+03 \n", + " Updated|Capacity|Electricity|Solar bla 2.556775e+04 \n", + " Updated|Capacity|Electricity|Wind bla 6.821745e+03 \n", + " Updated|Secondary Energy|Electricity|Biomass bla 1.173833e+06 \n", + " Updated|Secondary Energy|Electricity|Coal bla 5.027778e+03 \n", + " Updated|Secondary Energy|Electricity|Gas bla 2.602278e+06 \n", + " Updated|Secondary Energy|Electricity|Geothermal bla 6.177778e+05 \n", + " Updated|Secondary Energy|Electricity|Hydro bla 6.604167e+06 \n", + " Updated|Secondary Energy|Electricity|Nuclear bla 1.502778e+06 \n", + " Updated|Secondary Energy|Electricity|Oil bla 4.797222e+04 \n", + " Updated|Secondary Energy|Electricity|Other bla 1.985111e+06 \n", + " Updated|Secondary Energy|Electricity|Solar bla 3.404253e+07 \n", + " Updated|Secondary Energy|Electricity|Wind bla 1.727517e+07 \n", + "\n", + " 2045 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Updated|Capacity|Electricity|Biomass bla 1.063772e+02 \n", + " Updated|Capacity|Electricity|Coal bla 7.763000e-01 \n", + " Updated|Capacity|Electricity|Gas bla 8.638716e+02 \n", + " Updated|Capacity|Electricity|Geothermal bla 7.914520e+01 \n", + " Updated|Capacity|Electricity|Hydro bla 1.925653e+03 \n", + " Updated|Capacity|Electricity|Nuclear bla 1.860849e+02 \n", + " Updated|Capacity|Electricity|Oil bla 1.450230e+01 \n", + " Updated|Capacity|Electricity|Other bla 3.341699e+03 \n", + " Updated|Capacity|Electricity|Solar bla 3.097003e+04 \n", + " Updated|Capacity|Electricity|Wind bla 7.986305e+03 \n", + " Updated|Secondary Energy|Electricity|Biomass bla 1.359639e+06 \n", + " Updated|Secondary Energy|Electricity|Coal bla 3.694444e+03 \n", + " Updated|Secondary Energy|Electricity|Gas bla 1.752056e+06 \n", + " Updated|Secondary Energy|Electricity|Geothermal bla 5.893056e+05 \n", + " Updated|Secondary Energy|Electricity|Hydro bla 6.717417e+06 \n", + " Updated|Secondary Energy|Electricity|Nuclear bla 1.304083e+06 \n", + " Updated|Secondary Energy|Electricity|Oil bla 3.175000e+04 \n", + " Updated|Secondary Energy|Electricity|Other bla 2.790639e+06 \n", + " Updated|Secondary Energy|Electricity|Solar bla 4.088394e+07 \n", + " Updated|Secondary Energy|Electricity|Wind bla 2.112919e+07 \n", + "\n", + " 2050 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Updated|Capacity|Electricity|Biomass bla 8.602650e+01 \n", + " Updated|Capacity|Electricity|Coal bla 5.029000e-01 \n", + " Updated|Capacity|Electricity|Gas bla 5.628482e+02 \n", + " Updated|Capacity|Electricity|Geothermal bla 6.966280e+01 \n", + " Updated|Capacity|Electricity|Hydro bla 1.933520e+03 \n", + " Updated|Capacity|Electricity|Nuclear bla 1.567766e+02 \n", + " Updated|Capacity|Electricity|Oil bla 5.341200e+00 \n", + " Updated|Capacity|Electricity|Other bla 4.267072e+03 \n", + " Updated|Capacity|Electricity|Solar bla 3.608447e+04 \n", + " Updated|Capacity|Electricity|Wind bla 9.064963e+03 \n", + " Updated|Secondary Energy|Electricity|Biomass bla 1.471139e+06 \n", + " Updated|Secondary Energy|Electricity|Coal bla 2.416667e+03 \n", + " Updated|Secondary Energy|Electricity|Gas bla 1.218361e+06 \n", + " Updated|Secondary Energy|Electricity|Geothermal bla 5.187222e+05 \n", + " Updated|Secondary Energy|Electricity|Hydro bla 6.770250e+06 \n", + " Updated|Secondary Energy|Electricity|Nuclear bla 1.098694e+06 \n", + " Updated|Secondary Energy|Electricity|Oil bla 1.169444e+04 \n", + " Updated|Secondary Energy|Electricity|Other bla 3.488083e+06 \n", + " Updated|Secondary Energy|Electricity|Solar bla 4.723214e+07 \n", + " Updated|Secondary Energy|Electricity|Wind bla 2.367689e+07 \n", + "\n", + " 2055 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Updated|Capacity|Electricity|Biomass bla 6.565480e+01 \n", + " Updated|Capacity|Electricity|Coal bla 3.111000e-01 \n", + " Updated|Capacity|Electricity|Gas bla 3.937209e+02 \n", + " Updated|Capacity|Electricity|Geothermal bla 5.820210e+01 \n", + " Updated|Capacity|Electricity|Hydro bla 1.931110e+03 \n", + " Updated|Capacity|Electricity|Nuclear bla 1.239472e+02 \n", + " Updated|Capacity|Electricity|Oil bla 3.064300e+00 \n", + " Updated|Capacity|Electricity|Other bla 4.925403e+03 \n", + " Updated|Capacity|Electricity|Solar bla 4.110399e+04 \n", + " Updated|Capacity|Electricity|Wind bla 9.934779e+03 \n", + " Updated|Secondary Energy|Electricity|Biomass bla 1.513056e+06 \n", + " Updated|Secondary Energy|Electricity|Coal bla 1.555556e+03 \n", + " Updated|Secondary Energy|Electricity|Gas bla 6.538889e+05 \n", + " Updated|Secondary Energy|Electricity|Geothermal bla 4.333611e+05 \n", + " Updated|Secondary Energy|Electricity|Hydro bla 6.782028e+06 \n", + " Updated|Secondary Energy|Electricity|Nuclear bla 8.686111e+05 \n", + " Updated|Secondary Energy|Electricity|Oil bla 6.722222e+03 \n", + " Updated|Secondary Energy|Electricity|Other bla 4.020806e+06 \n", + " Updated|Secondary Energy|Electricity|Solar bla 5.327061e+07 \n", + " Updated|Secondary Energy|Electricity|Wind bla 2.569986e+07 \n", + "\n", + " 2060 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Updated|Capacity|Electricity|Biomass bla 4.575910e+01 \n", + " Updated|Capacity|Electricity|Coal bla 1.836000e-01 \n", + " Updated|Capacity|Electricity|Gas bla 3.499091e+02 \n", + " Updated|Capacity|Electricity|Geothermal bla 4.840070e+01 \n", + " Updated|Capacity|Electricity|Hydro bla 1.922569e+03 \n", + " Updated|Capacity|Electricity|Nuclear bla 9.206670e+01 \n", + " Updated|Capacity|Electricity|Oil bla 1.200000e-02 \n", + " Updated|Capacity|Electricity|Other bla 5.433398e+03 \n", + " Updated|Capacity|Electricity|Solar bla 4.527457e+04 \n", + " Updated|Capacity|Electricity|Wind bla 1.082886e+04 \n", + " Updated|Secondary Energy|Electricity|Biomass bla 1.544750e+06 \n", + " Updated|Secondary Energy|Electricity|Coal bla 9.444444e+02 \n", + " Updated|Secondary Energy|Electricity|Gas bla 5.281944e+05 \n", + " Updated|Secondary Energy|Electricity|Geothermal bla 3.603889e+05 \n", + " Updated|Secondary Energy|Electricity|Hydro bla 6.775917e+06 \n", + " Updated|Secondary Energy|Electricity|Nuclear bla 6.451944e+05 \n", + " Updated|Secondary Energy|Electricity|Oil bla 2.777778e+01 \n", + " Updated|Secondary Energy|Electricity|Other bla 4.434639e+06 \n", + " Updated|Secondary Energy|Electricity|Solar bla 5.819236e+07 \n", + " Updated|Secondary Energy|Electricity|Wind bla 2.771675e+07 \n", + "\n", + " 2070 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Updated|Capacity|Electricity|Biomass bla 1.450470e+01 \n", + " Updated|Capacity|Electricity|Coal bla 7.830000e-02 \n", + " Updated|Capacity|Electricity|Gas bla 3.063883e+02 \n", + " Updated|Capacity|Electricity|Geothermal bla 3.927950e+01 \n", + " Updated|Capacity|Electricity|Hydro bla 1.893641e+03 \n", + " Updated|Capacity|Electricity|Nuclear bla 3.877600e+01 \n", + " Updated|Capacity|Electricity|Oil bla 1.200000e-02 \n", + " Updated|Capacity|Electricity|Other bla 6.432168e+03 \n", + " Updated|Capacity|Electricity|Solar bla 5.443399e+04 \n", + " Updated|Capacity|Electricity|Wind bla 1.320022e+04 \n", + " Updated|Secondary Energy|Electricity|Biomass bla 1.564583e+06 \n", + " Updated|Secondary Energy|Electricity|Coal bla 4.444444e+02 \n", + " Updated|Secondary Energy|Electricity|Gas bla 2.423333e+05 \n", + " Updated|Secondary Energy|Electricity|Geothermal bla 2.924722e+05 \n", + " Updated|Secondary Energy|Electricity|Hydro bla 6.722667e+06 \n", + " Updated|Secondary Energy|Electricity|Nuclear bla 2.717500e+05 \n", + " Updated|Secondary Energy|Electricity|Oil bla 2.777778e+01 \n", + " Updated|Secondary Energy|Electricity|Other bla 5.261472e+06 \n", + " Updated|Secondary Energy|Electricity|Solar bla 6.826367e+07 \n", + " Updated|Secondary Energy|Electricity|Wind bla 3.280081e+07 \n", + "\n", + " 2080 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Updated|Capacity|Electricity|Biomass bla 3.962800e+00 \n", + " Updated|Capacity|Electricity|Coal bla 9.180000e-02 \n", + " Updated|Capacity|Electricity|Gas bla 2.135970e+02 \n", + " Updated|Capacity|Electricity|Geothermal bla 3.347930e+01 \n", + " Updated|Capacity|Electricity|Hydro bla 1.860861e+03 \n", + " Updated|Capacity|Electricity|Nuclear bla 8.455000e+00 \n", + " Updated|Capacity|Electricity|Oil bla 1.200000e-02 \n", + " Updated|Capacity|Electricity|Other bla 7.478326e+03 \n", + " Updated|Capacity|Electricity|Solar bla 6.305545e+04 \n", + " Updated|Capacity|Electricity|Wind bla 1.459313e+04 \n", + " Updated|Secondary Energy|Electricity|Biomass bla 1.557556e+06 \n", + " Updated|Secondary Energy|Electricity|Coal bla 5.277778e+02 \n", + " Updated|Secondary Energy|Electricity|Gas bla 1.685833e+05 \n", + " Updated|Secondary Energy|Electricity|Geothermal bla 2.492778e+05 \n", + " Updated|Secondary Energy|Electricity|Hydro bla 6.655833e+06 \n", + " Updated|Secondary Energy|Electricity|Nuclear bla 5.925000e+04 \n", + " Updated|Secondary Energy|Electricity|Oil bla 2.777778e+01 \n", + " Updated|Secondary Energy|Electricity|Other bla 6.099583e+06 \n", + " Updated|Secondary Energy|Electricity|Solar bla 7.754106e+07 \n", + " Updated|Secondary Energy|Electricity|Wind bla 3.570786e+07 \n", + "\n", + " 2090 \\\n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Updated|Capacity|Electricity|Biomass bla 8.198000e-01 \n", + " Updated|Capacity|Electricity|Coal bla 9.650000e-02 \n", + " Updated|Capacity|Electricity|Gas bla 5.260450e+01 \n", + " Updated|Capacity|Electricity|Geothermal bla 2.765300e+01 \n", + " Updated|Capacity|Electricity|Hydro bla 1.846102e+03 \n", + " Updated|Capacity|Electricity|Nuclear bla 5.580000e-01 \n", + " Updated|Capacity|Electricity|Oil bla 1.210000e-02 \n", + " Updated|Capacity|Electricity|Other bla 8.321384e+03 \n", + " Updated|Capacity|Electricity|Solar bla 6.884808e+04 \n", + " Updated|Capacity|Electricity|Wind bla 1.601071e+04 \n", + " Updated|Secondary Energy|Electricity|Biomass bla 1.543056e+06 \n", + " Updated|Secondary Energy|Electricity|Coal bla 5.555556e+02 \n", + " Updated|Secondary Energy|Electricity|Gas bla 4.166667e+04 \n", + " Updated|Secondary Energy|Electricity|Geothermal bla 2.059167e+05 \n", + " Updated|Secondary Energy|Electricity|Hydro bla 6.631806e+06 \n", + " Updated|Secondary Energy|Electricity|Nuclear bla 3.916667e+03 \n", + " Updated|Secondary Energy|Electricity|Oil bla 2.777778e+01 \n", + " Updated|Secondary Energy|Electricity|Other bla 6.740833e+06 \n", + " Updated|Secondary Energy|Electricity|Solar bla 8.359456e+07 \n", + " Updated|Secondary Energy|Electricity|Wind bla 3.840797e+07 \n", + "\n", + " 2100 \n", + "model scenario region variable unit \n", + "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 World Updated|Capacity|Electricity|Biomass bla 2.150000e-01 \n", + " Updated|Capacity|Electricity|Coal bla 8.660000e-02 \n", + " Updated|Capacity|Electricity|Gas bla 2.717000e-01 \n", + " Updated|Capacity|Electricity|Geothermal bla 2.601980e+01 \n", + " Updated|Capacity|Electricity|Hydro bla 1.805140e+03 \n", + " Updated|Capacity|Electricity|Nuclear bla 1.900000e-02 \n", + " Updated|Capacity|Electricity|Oil bla 1.260000e-02 \n", + " Updated|Capacity|Electricity|Other bla 9.334424e+03 \n", + " Updated|Capacity|Electricity|Solar bla 7.743017e+04 \n", + " Updated|Capacity|Electricity|Wind bla 1.736882e+04 \n", + " Updated|Secondary Energy|Electricity|Biomass bla 1.574306e+06 \n", + " Updated|Secondary Energy|Electricity|Coal bla 5.000000e+02 \n", + " Updated|Secondary Energy|Electricity|Gas bla 3.888889e+02 \n", + " Updated|Secondary Energy|Electricity|Geothermal bla 1.937500e+05 \n", + " Updated|Secondary Energy|Electricity|Hydro bla 6.533167e+06 \n", + " Updated|Secondary Energy|Electricity|Nuclear bla 1.388889e+02 \n", + " Updated|Secondary Energy|Electricity|Oil bla 2.777778e+01 \n", + " Updated|Secondary Energy|Electricity|Other bla 7.547194e+06 \n", + " Updated|Secondary Energy|Electricity|Solar bla 9.270394e+07 \n", + " Updated|Secondary Energy|Electricity|Wind bla 4.123369e+07 " ] }, - "execution_count": 18, + "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "from pandas_indexing import projectlevel\n", + "from pandas_indexing import assignlevel\n", "\n", - "simple_fossil_series = projectlevel(fossil_series, [\"variable\", \"year\"])\n", - "simple_fossil_series" + "assignlevel(df, variable=\"Updated|\" + projectlevel(df.index, \"variable\"), unit=\"bla\")\n", + "# or: df.idx.assign(variable=df.index.idx.project(\"variable\"), unit=\"bla\")" ] }, { "attachments": {}, "cell_type": "markdown", - "id": "715fe070", - "metadata": {}, - "source": [ - "`projectlevel` reduces the levels attached to a multiindex to the ones explicitly named. It is basically the complement to `droplevel` which removes the listed names" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "id": "308fb920", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "model scenario \n", - "REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 True\n", - " DeepElec_SSP2_HighRE_Budg900 True\n", - " DeepElec_SSP2_HighRE_Budg900 True\n", - " DeepElec_SSP2_HighRE_Budg900 True\n", - " DeepElec_SSP2_HighRE_Budg900 True\n", - " DeepElec_SSP2_HighRE_Budg900 True\n", - " DeepElec_SSP2_HighRE_Budg900 True\n", - " DeepElec_SSP2_HighRE_Budg900 True\n", - " DeepElec_SSP2_HighRE_Budg900 True\n", - " DeepElec_SSP2_HighRE_Budg900 True\n", - " DeepElec_SSP2_HighRE_Budg900 True\n", - " DeepElec_SSP2_HighRE_Budg900 True\n", - "dtype: bool" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "projectlevel(fossil_series, [\"model\", \"scenario\"]) == fossil_series.droplevel(\n", - " [\"variable\", \"unit\", \"region\", \"year\"]\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "6eb7b029", - "metadata": {}, - "source": [ - "## Assigning to levels\n", - "\n", - "`assignlevel` allows to modify individual values with helpful keyword arguments:" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "id": "4f52e21a", + "id": "1349c89d", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Index(['Updated|Capacity|Electricity|Biomass',\n", - " 'Updated|Capacity|Electricity|Coal', 'Updated|Capacity|Electricity|Gas',\n", - " 'Updated|Capacity|Electricity|Geothermal',\n", - " 'Updated|Capacity|Electricity|Hydro',\n", - " 'Updated|Capacity|Electricity|Nuclear',\n", - " 'Updated|Capacity|Electricity|Oil',\n", - " 'Updated|Capacity|Electricity|Other',\n", - " 'Updated|Capacity|Electricity|Solar',\n", - " 'Updated|Capacity|Electricity|Wind',\n", - " 'Updated|Secondary Energy|Electricity|Biomass',\n", - " 'Updated|Secondary Energy|Electricity|Coal',\n", - " 'Updated|Secondary Energy|Electricity|Gas',\n", - " 'Updated|Secondary Energy|Electricity|Geothermal',\n", - " 'Updated|Secondary Energy|Electricity|Hydro',\n", - " 'Updated|Secondary Energy|Electricity|Nuclear',\n", - " 'Updated|Secondary Energy|Electricity|Oil',\n", - " 'Updated|Secondary Energy|Electricity|Other',\n", - " 'Updated|Secondary Energy|Electricity|Solar',\n", - " 'Updated|Secondary Energy|Electricity|Wind'],\n", - " dtype='object', name='variable')" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ - "\"Updated|\" + projectlevel(df.index, \"variable\")" + "This particular case is even more clearly handled with `formatlevel`:" ] }, { "cell_type": "code", - "execution_count": 21, - "id": "ca2a3c70", + "execution_count": 24, + "id": "f4cb55c2", "metadata": {}, "outputs": [ { @@ -2688,28 +4580,30 @@ " Updated|Secondary Energy|Electricity|Wind bla 4.123369e+07 " ] }, - "execution_count": 21, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "from pandas_indexing import assignlevel\n", + "from pandas_indexing import formatlevel\n", "\n", - "assignlevel(df, variable=\"Updated|\" + projectlevel(df.index, \"variable\"), unit=\"bla\")" + "formatlevel(df, variable=\"Updated|{variable}\", unit=\"bla\")\n", + "# or: df.idx.format(variable=...)" ] }, { + "attachments": {}, "cell_type": "markdown", "id": "a8416465", "metadata": {}, "source": [ - "and as such avoids having to rely on `reset_index`, `set_index` pairs, which are painful for large data, since `set_index` is expensive!" + "Both functions avoid having to rely on `reset_index`, `set_index` pairs, which are painful for large data, since `set_index` is expensive!" ] }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 25, "id": "72d60d5e", "metadata": {}, "outputs": [ @@ -3538,7 +5432,7 @@ " GWh/yr 4.123369e+07 " ] }, - "execution_count": 22, + "execution_count": 25, "metadata": {}, "output_type": "execute_result" } @@ -3548,14 +5442,142 @@ ] }, { + "attachments": {}, + "cell_type": "markdown", + "id": "63103fb7", + "metadata": {}, + "source": [ + "# Examining level values and level combinations\n", + "\n", + "We already encountered the possibility to get an overview of the available levels and their values with describelevel:" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "029603e9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Index:\n", + " * model : REMIND-MAgPIE 2.1-4.3 (1)\n", + " * scenario : DeepElec_SSP2_HighRE_Budg900 (1)\n", + " * region : World (1)\n", + " * variable : Capacity|Electricity|Biomass, ... (20)\n", + " * unit : GW, GWh/yr (2)\n", + "\n", + "Columns:\n", + " * : 2005, 2010, 2015, 2020, 2025, 2030, 2035, 2040, ... 2100 (16)\n" + ] + } + ], + "source": [ + "describelevel(df) # or: df.idx" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "5938866f", + "metadata": {}, + "source": [ + "Often it is necessary to get programmatic access to the unique values of one or more levels:" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "f9fe7174", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Index(['Capacity|Electricity|Biomass', 'Capacity|Electricity|Coal',\n", + " 'Capacity|Electricity|Gas', 'Capacity|Electricity|Geothermal',\n", + " 'Capacity|Electricity|Hydro', 'Capacity|Electricity|Nuclear',\n", + " 'Capacity|Electricity|Oil', 'Capacity|Electricity|Other',\n", + " 'Capacity|Electricity|Solar', 'Capacity|Electricity|Wind',\n", + " 'Secondary Energy|Electricity|Biomass',\n", + " 'Secondary Energy|Electricity|Coal', 'Secondary Energy|Electricity|Gas',\n", + " 'Secondary Energy|Electricity|Geothermal',\n", + " 'Secondary Energy|Electricity|Hydro',\n", + " 'Secondary Energy|Electricity|Nuclear',\n", + " 'Secondary Energy|Electricity|Oil',\n", + " 'Secondary Energy|Electricity|Other',\n", + " 'Secondary Energy|Electricity|Solar',\n", + " 'Secondary Energy|Electricity|Wind'],\n", + " dtype='object', name='variable')" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from pandas_indexing import uniquelevel\n", + "\n", + "uniquelevel(df, \"variable\")\n", + "# or: df.idx.unique(\"variable\")\n", + "# or in vanilla pandas: df.index.unique(\"variable\")" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "23378320", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "MultiIndex([( 'Capacity|Electricity|Biomass', 'GW'),\n", + " ( 'Capacity|Electricity|Coal', 'GW'),\n", + " ( 'Capacity|Electricity|Gas', 'GW'),\n", + " ( 'Capacity|Electricity|Geothermal', 'GW'),\n", + " ( 'Capacity|Electricity|Hydro', 'GW'),\n", + " ( 'Capacity|Electricity|Nuclear', 'GW'),\n", + " ( 'Capacity|Electricity|Oil', 'GW'),\n", + " ( 'Capacity|Electricity|Other', 'GW'),\n", + " ( 'Capacity|Electricity|Solar', 'GW'),\n", + " ( 'Capacity|Electricity|Wind', 'GW'),\n", + " ( 'Secondary Energy|Electricity|Biomass', 'GWh/yr'),\n", + " ( 'Secondary Energy|Electricity|Coal', 'GWh/yr'),\n", + " ( 'Secondary Energy|Electricity|Gas', 'GWh/yr'),\n", + " ('Secondary Energy|Electricity|Geothermal', 'GWh/yr'),\n", + " ( 'Secondary Energy|Electricity|Hydro', 'GWh/yr'),\n", + " ( 'Secondary Energy|Electricity|Nuclear', 'GWh/yr'),\n", + " ( 'Secondary Energy|Electricity|Oil', 'GWh/yr'),\n", + " ( 'Secondary Energy|Electricity|Other', 'GWh/yr'),\n", + " ( 'Secondary Energy|Electricity|Solar', 'GWh/yr'),\n", + " ( 'Secondary Energy|Electricity|Wind', 'GWh/yr')],\n", + " names=['variable', 'unit'])" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "uniquelevel(df, [\"variable\", \"unit\"])" + ] + }, + { + "attachments": {}, "cell_type": "markdown", "id": "c84e2a30-1880-4687-b450-6c5b4dac3c01", "metadata": {}, "source": [ - "## BEWARE: Pitfalls" + "# BEWARE: Pitfalls" ] }, { + "attachments": {}, "cell_type": "markdown", "id": "ec137242", "metadata": {}, @@ -3565,7 +5587,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 29, "id": "24d42af8-ffd3-4281-9d75-1449046fd6fd", "metadata": {}, "outputs": [ @@ -3600,7 +5622,7 @@ "dtype: float64" ] }, - "execution_count": 23, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } @@ -3611,7 +5633,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 30, "id": "254197c5-8498-4c69-a30d-044e7edb7b53", "metadata": {}, "outputs": [ @@ -3646,7 +5668,7 @@ "dtype: float64" ] }, - "execution_count": 24, + "execution_count": 30, "metadata": {}, "output_type": "execute_result" } @@ -3674,52 +5696,7 @@ ] }, { - "cell_type": "code", - "execution_count": 25, - "id": "a8300699", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "MultiIndex([('REMIND-MAgPIE 2.1-4.3', 2030),\n", - " ('REMIND-MAgPIE 2.1-4.3', 2040),\n", - " ('REMIND-MAgPIE 2.1-4.3', 2050),\n", - " ('REMIND-MAgPIE 2.1-4.3', 2060)],\n", - " names=['model', 'year'])" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "projectlevel(fossil_series, [\"model\", \"year\"]).index.unique()" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "id": "187e2a70", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Int64Index([2030, 2040, 2050, 2060], dtype='int64', name='year')" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "fossil_series.index.unique(\"year\")" - ] - }, - { + "attachments": {}, "cell_type": "markdown", "id": "46c57e84", "metadata": {}, @@ -3729,7 +5706,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 31, "id": "5b7cb7db", "metadata": {}, "outputs": [ @@ -3751,7 +5728,7 @@ " names=['model', 'scenario', 'variable'])" ] }, - "execution_count": 27, + "execution_count": 31, "metadata": {}, "output_type": "execute_result" } @@ -3762,7 +5739,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 32, "id": "5b7cb7db", "metadata": {}, "outputs": [ @@ -3885,7 +5862,7 @@ "11 REMIND-MAgPIE 2.1-4.3 DeepElec_SSP2_HighRE_Budg900 Nuclear" ] }, - "execution_count": 28, + "execution_count": 32, "metadata": {}, "output_type": "execute_result" } diff --git a/src/pandas_indexing/__init__.py b/src/pandas_indexing/__init__.py index 4f6ed8d..7535c4c 100644 --- a/src/pandas_indexing/__init__.py +++ b/src/pandas_indexing/__init__.py @@ -13,6 +13,8 @@ assignlevel, describelevel, dropnalevel, + extractlevel, + formatlevel, index_names, projectlevel, semijoin, diff --git a/src/pandas_indexing/accessors.py b/src/pandas_indexing/accessors.py index 7db1545..e2c5a15 100644 --- a/src/pandas_indexing/accessors.py +++ b/src/pandas_indexing/accessors.py @@ -1,5 +1,4 @@ -""" -Registers convenience accessors into the ``idx`` namespace of each pandas +"""Registers convenience accessors into the ``idx`` namespace of each pandas object. Usage diff --git a/src/pandas_indexing/arithmetics.py b/src/pandas_indexing/arithmetics.py index 19c2509..5703d05 100644 --- a/src/pandas_indexing/arithmetics.py +++ b/src/pandas_indexing/arithmetics.py @@ -1,5 +1,4 @@ -""" -Provide aligned basic arithmetic ops. +"""Provide aligned basic arithmetic ops. Simple arithmetic operations :py:func:`add`, :py:func:`divide`, :py:func:`multiply` and :py:func:`subtract` which allow setting the standard how="outer" alignment that pandas diff --git a/src/pandas_indexing/core.py b/src/pandas_indexing/core.py index 97ade3c..53e4bba 100644 --- a/src/pandas_indexing/core.py +++ b/src/pandas_indexing/core.py @@ -1,10 +1,11 @@ """ Core module. """ +import re from functools import reduce from itertools import chain from operator import and_, or_ -from typing import Any, Literal, Optional, Sequence, TypeVar, Union +from typing import Any, Literal, Optional, Sequence, Tuple, TypeVar, Union import numpy as np from deprecated import deprecated @@ -75,13 +76,12 @@ def assignlevel( axis: Axis = 0, **labels: Any, ) -> T: - """ - Add or overwrite levels on a multiindex. + """Add or overwrite levels on a multiindex. Parameters ----------\ {df} - frame : Series|DataFrame, optional + frame : Series or DataFrame, optional Additional labels order : list of str, optional Level names in desired order or False, by default False @@ -118,8 +118,7 @@ def _projectlevel(index: Index, levels: Sequence[str]) -> Index: """ ) def projectlevel(index_or_data: T, levels: Sequence[str], axis: Axis = 0) -> T: - """ - Project multiindex to given `levels` + """Project multiindex to given ``levels``. Drops all levels except the ones explicitly mentioned from a given multiindex or an axis of a series or a dataframe. @@ -127,7 +126,7 @@ def projectlevel(index_or_data: T, levels: Sequence[str], axis: Axis = 0) -> T: Parameters ----------\ {index_or_data} - levels : Sequence[str] + levels : sequence of str Names of levels to project on (to keep) axis : {{0, 1, "index", "columns"}}, default 0 Axis of DataFrame to project @@ -154,7 +153,8 @@ def _notna( subset: Optional[Sequence[str]] = None, how: Literal["any", "all"] = "any", ) -> np.ndarray: - index = ensure_multiindex(index) + if not isinstance(index, MultiIndex): + return index.notna() subset = index.names if subset is None else np.atleast_1d(subset) codes = [index.codes[index.names.index(n)] for n in subset] @@ -162,6 +162,24 @@ def _notna( return reduce(op, [c != -1 for c in codes]) +def notna( + index_or_data: Union[Index, Series, DataFrame], + subset: Optional[Sequence[str]] = None, + how: Literal["any", "all"] = "any", + axis: Axis = 0, +): + return _notna(get_axis(index_or_data, axis), subset, how) + + +def isna( + index_or_data: Union[Index, Series, DataFrame], + subset: Optional[Sequence[str]] = None, + how: Literal["any", "all"] = "any", + axis: Axis = 0, +): + return ~_notna(get_axis(index_or_data, axis), subset, how) + + @doc( index_or_data=""" index_or_data : DataFrame, Series or Index @@ -174,10 +192,9 @@ def dropnalevel( how: Literal["any", "all"] = "any", axis: Axis = 0, ) -> T: - """ - Remove missing index values. + """Remove missing index values. - Drops all index entries for which any or all (`how`) levels are + Drops all index entries for which any or all (``how``) levels are undefined. Parameters @@ -185,8 +202,8 @@ def dropnalevel( {index_or_data} subset : Sequence[str], optional Names of levels on which to check for NA values - how : "any" (default) or "all" - Whether to remove an entry if all levels are NA only a single one + how : {{"any", "all"}} + Whether to remove an entry if all levels are NA or only a single one axis : {{0, 1, "index", "columns"}}, default 0 Axis of DataFrame to check on @@ -220,8 +237,7 @@ def uniquelevel( levels: Union[str, Sequence[str], None], axis: Axis = 0, ) -> Index: - """ - Return unique index levels. + """Return unique index levels. Parameters ----------\ @@ -271,8 +287,7 @@ def name(l): def describelevel( index_or_data: Union[DataFrame, Series, Index], n: int = 80, as_str: bool = False ) -> Optional[str]: - """ - Describe index levels. + """Describe index levels. Parameters ----------\ @@ -285,7 +300,7 @@ def describelevel( Returns ------- description : str, optional - if print is False + if as_str is True See also -------- @@ -376,7 +391,7 @@ def alignlevels(l, r): @doc( frame_or_series=""" frame_or_series : DataFrame or Series - data to be filtered\ + Data to be filtered\ """ ) def semijoin( @@ -388,20 +403,19 @@ def semijoin( sort: bool = False, axis: Axis = 0, ) -> S: - """ - Semijoin ``data`` by index ``other`` + """Semijoin ``data`` by index ``other``. Parameters ----------\ {frame_or_series} other : Index - other index to join with + Other index to join with how : {{'left', 'right', 'inner', 'outer'}} Join method to use level : None or str or int or - single level on which to join, if not given join on all + Single level on which to join, if not given join on all sort : bool, optional - whether to sort the index + Whether to sort the index axis : {{0, 1, "index", "columns"}} Axis on which to join @@ -463,3 +477,153 @@ def semijoin( ) return cls(data, *axes).__finalize__(frame_or_series) + + +def _extractlevel( + index: Index, drop: bool = False, **templates: str +) -> Tuple[Index, list[str]]: + index = ensure_multiindex(index) + all_identifiers = set() + + for dim, template in templates.items(): + identifiers = re.findall(r"\{([a-zA-Z_]+)\}", template) + all_identifiers.update(identifiers) + if dim not in index.names: + raise ValueError(f"{dim} not a dimension of index: {index.names}") + + levelnum = index.names.index(dim) + labels = index.levels[levelnum] + codes = index.codes[levelnum] + + regex_pattern = reduce( + lambda s, ident: s.replace(rf"\{{{ident}\}}", rf"(?P<{ident}>.*?)"), + identifiers, + re.escape(template), + ) + components = labels.str.extract(f"^{regex_pattern}$", expand=True) + + index = assignlevel( + index, **{ident: components[ident].values[codes] for ident in identifiers} + ) + + if drop: + index = index.droplevel(list(set(templates) - all_identifiers)) + + return index, list(all_identifiers) + + +@doc( + index_or_data=""" + index_or_data : DataFrame, Series or Index + Data to modify\ + """ +) +def extractlevel( + index_or_data: T, + drop: bool = False, + dropna: bool = True, + axis: Axis = 0, + **templates: str, +) -> T: + """Split an index ``dim`` ension into multiple ones based on a + ``template``. + + Parameters + ----------\ + {index_or_data} + drop : bool, default False + Whether to keep the split dimension + dropna : bool, default True + Whether to drop the non-matching levels + axis : {{0, 1, "index", "columns"}}, default 0 + Axis of DataFrame to extract from + **templates : str + Templates for splitting one or multiple levels + + Returns + ------- + Index, Series or DataFrame + + Raises + ------ + ValueError + If ``dim`` is not a dimension of ``index_or_series`` + """ + if isinstance(index_or_data, Index): + index_or_data, identifiers = _extractlevel(index_or_data, drop, **templates) + else: + index, identifiers = _extractlevel( + get_axis(index_or_data, axis), drop, **templates + ) + index_or_data = index_or_data.set_axis(index, axis=axis) + + if dropna: + index_or_data = dropnalevel(index_or_data, subset=identifiers, axis=axis) + + return index_or_data + + +def _formatlevel(index: Index, drop: bool = False, **templates: str) -> Index: + levels = {} + used_levels = set() + for dim, template in templates.items(): + # Build string + string = "" + prev_end = 0 + for m in re.finditer(r"\{([a-zA-Z_]+)\}", template): + level = m.group(1) + start, end = m.span() + string += template[prev_end:start] + projectlevel(index, level).astype(str) + prev_end = end + used_levels.add(level) + string += template[prev_end:] + + levels[dim] = string + + if drop: + used_levels.difference_update(templates) + if used_levels: + index = index.droplevel(list(used_levels)) + + return assignlevel(index, **levels) + + +@doc( + index_or_data=""" + index_or_data : DataFrame, Series or Index + Data to modify\ + """ +) +def formatlevel( + index_or_data: T, + drop: bool = False, + axis: Axis = 0, + **templates: str, +) -> T: + """Format index levels based on a ``template`` which can refer to other + levels. + + Parameters + ----------\ + {index_or_data} + drop : bool, default False + Whether to drop the used index levels + axis : {{0, 1, "index", "columns"}}, default 0 + Axis of DataFrame to modify + **templates : str + Format templates for one or multiple levels + + Returns + ------- + Index, Series or DataFrame + + Raises + ------ + ValueError + If ``templates`` refer to non-existant levels + """ + if isinstance(index_or_data, Index): + return _formatlevel(index_or_data, drop, **templates) + + index = get_axis(index_or_data, axis) + return index_or_data.set_axis(_formatlevel(index, drop, **templates), axis=axis) diff --git a/src/pandas_indexing/datasets/__init__.py b/src/pandas_indexing/datasets/__init__.py index df549d0..0c94f74 100644 --- a/src/pandas_indexing/datasets/__init__.py +++ b/src/pandas_indexing/datasets/__init__.py @@ -8,8 +8,7 @@ def remindhighre_power(): - """ - Reads IAMC power sector data from REMIND's HighRE IMP scenario for AR6. + """Read IAMC power sector data from REMIND's HighRE IMP scenario for AR6. Returns ------- diff --git a/src/pandas_indexing/selectors.py b/src/pandas_indexing/selectors.py index e5c662e..f8dfa81 100644 --- a/src/pandas_indexing/selectors.py +++ b/src/pandas_indexing/selectors.py @@ -1,5 +1,5 @@ """ -Selectors improve .loc[] indexing for multi-index pandas data. +Selectors improve ``.loc[]`` indexing for multi-index pandas data. """ from functools import reduce @@ -91,24 +91,23 @@ def __call__(self, df): def isin( df: Optional[Data] = None, ignore_missing_levels: bool = False, **filters: Any ) -> Union[Isin, Series]: - """ - Constructs a MultiIndex selector. + """Constructs a MultiIndex selector. Arguments --------- df : Data, optional - Data on which to match, if missing an Isin object is returned + Data on which to match, if missing an ``Isin`` object is returned ignore_missing_levels : bool, default False If set, levels missing in data index will be ignored **filters - Filter to apply on given levels (lists are `or`ed, levels are `and`ed) + Filter to apply on given levels (lists are ``or`` ed, levels are ``and`` ed) Returns ------- Isin or Series - Usage - ----- + Example + ------- >>> df.loc[isin(region="World", gas=["CO2", "N2O"])] or with explicit df to get a boolean mask @@ -176,15 +175,14 @@ def ismatch( ignore_missing_levels: bool = False, **filters, ) -> Union[Ismatch, Series]: - """ - Constructs an Index or MultiIndex selector based on pattern matching. + """Constructs an Index or MultiIndex selector based on pattern matching. Arguments --------- df : Data, optional - Data on which to match, if missing an Isin object is returned. + Data on which to match, if missing an ``Isin`` object is returned. singlefilter : str, optional - Filter to apply on a non-multiindex index (can also be handed into the `df` + Filter to apply on a non-multiindex index (can also be handed into the ``df`` argument) regex : bool, default False If set, filters are interpreted as plain regex strings, otherwise (by default) a @@ -192,14 +190,14 @@ def ismatch( ignore_missing_levels : bool, default False If set, levels missing in data index will be ignored **filters - Filter to apply on given levels (lists are `or`ed, levels are `and`ed) + Filter to apply on given levels (lists are ``or`` ed, levels are ``and`` ed) Returns ------- Isin or Series - Usage - ----- + Example + ------- for a multiindex: >>> df.loc[ismatch(variable="Emissions|*|Fossil Fuel and Industry")] diff --git a/src/pandas_indexing/utils.py b/src/pandas_indexing/utils.py index 2894641..dfe3fce 100644 --- a/src/pandas_indexing/utils.py +++ b/src/pandas_indexing/utils.py @@ -1,5 +1,4 @@ -""" -Utils module. +"""Utils module. Simple utility functions not of greater interest """ @@ -31,8 +30,7 @@ def shell_pattern_to_regex(s): def print_list(x, n): - """ - Return a printable string of a list shortened to n characters. + """Return a printable string of a list shortened to n characters. Copied from pyam.utils.print_list by Daniel Huppmann, licensed under Apache 2.0.