Merge pull request #110 from OSeMOSYS/results

User config groundwork, CPLEX results and floating point numbers

.gitignore

@@ -13,9 +13,22 @@ __pycache__/*
 .*.swp
 */.ipynb_checkpoints/*
 .DS_Store
-.vscode
+*.feather
+*.parquet
+*.kgrind
+*.sol
+*.lp
+*.txt
+*.csv
+datapackage.json
+*.ipynb
 
 # Project files
+*.srctrlbm
+*.srctrldb
+*.srctrlprj
+.vscode
+*.code-workspace
 .ropeproject
 .project
 .pydevproject

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ exclude: '^docs/conf.py'
 
 repos:
 - repo: https://github.com/psf/black
-  rev: stable
+  rev: 21.5b1
   hooks:
     - id: black
       language_version: python3.8
@@ -26,11 +26,11 @@ repos:
     args: ['--max-line-length=88','--extend-ignore=E203']  # default of Black
 
 - repo: https://github.com/pre-commit/mirrors-isort
-  rev: v4.3.4
+  rev: v5.8.0
   hooks:
   - id: isort
 
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: master  # Use the sha / tag you want to point at
+    rev: v0.812  # Use the sha / tag you want to point at
     hooks:
     -   id: mypy

src/otoole/cli.py

@@ -41,6 +41,7 @@
 """
 import argparse
 import logging
+import os
 import sys
 
 from otoole import (
@@ -58,7 +59,7 @@
     __version__,
 )
 from otoole.input import Context
-from otoole.utils import read_packaged_file
+from otoole.utils import _read_file, read_packaged_file
 from otoole.validate import main as validate
 from otoole.visualise import create_res
 
@@ -141,14 +142,22 @@ def conversion_matrix(args):
     read_strategy = None
     write_strategy = None
 
+    config = None
+    if args.config:
+        for filename in args.config:
+            _, ending = os.path.splitext(filename)
+            with open(filename, "r") as config_file:
+                config = _read_file(config_file, ending)
+            logger.info("Reading config from {}".format(filename))
+
     if args.from_format == "datafile":
-        read_strategy = ReadDatafile()
+        read_strategy = ReadDatafile(user_config=config)
     elif args.from_format == "datapackage":
-        read_strategy = ReadDatapackage()
+        read_strategy = ReadDatapackage(user_config=config)
     elif args.from_format == "csv":
-        read_strategy = ReadCsv()
+        read_strategy = ReadCsv(user_config=config)
     elif args.from_format == "excel":
-        read_strategy = ReadExcel()
+        read_strategy = ReadExcel(user_config=config)
 
     if args.to_format == "datapackage":
         write_strategy = WriteDatapackage()
@@ -232,6 +241,9 @@ def get_parser():
         "from_path", help="Path to file or folder to convert from"
     )
     convert_parser.add_argument("to_path", help="Path to file or folder to convert to")
+    convert_parser.add_argument(
+        "-c", "--config", action="append", help="Path to config YAML files"
+    )
     convert_parser.set_defaults(func=conversion_matrix)
 
     # Parser for validation
@@ -279,7 +291,7 @@ def exception_handler(
         if args.verbose:
             debug_hook(exception_type, exception, traceback)
         else:
-            print("{}: {}".format(exception_type.__name__, exception.message))
+            print("{}: {}".format(exception_type.__name__, str(exception)))
 
     sys.excepthook = exception_handler
 

src/otoole/input.py

@@ -42,7 +42,7 @@
 
 import logging
 from abc import ABC, abstractmethod
-from typing import Any, Dict, Optional, TextIO, Tuple, Union
+from typing import Any, Dict, List, Optional, TextIO, Tuple, Union
 
 import pandas as pd
 
@@ -138,18 +138,17 @@ class Strategy(ABC):
     def __init__(
         self, user_config: Optional[Dict] = None, results_config: Optional[Dict] = None
     ):
-        self._input_config = {}
+        self._input_config = {}  # type: Dict[str, Dict[str, Union[str, List[str]]]]
         self._results_config = {}
 
         if user_config:
-            self._input_config = user_config
+            self.input_config = user_config
         else:
-            self._input_config = self._read_config()
+            self.input_config = self._read_config()
         if results_config:
             self._results_config = results_config
         else:
             self._results_config = self._read_results_config()
-        self._input_config = self._add_dtypes(self._input_config)
 
     def _add_dtypes(self, config: Dict):
         for name, details in config.items():
@@ -170,9 +169,13 @@ def _read_results_config(self) -> Dict[str, Dict]:
         return read_packaged_file("config.yaml", "otoole.results")
 
     @property
-    def input_config(self):
+    def input_config(self) -> Dict:
         return self._input_config
 
+    @input_config.setter
+    def input_config(self, value: Dict):
+        self._input_config = self._add_dtypes(value)
+
     @property
     def results_config(self):
         return self._results_config
@@ -310,7 +313,7 @@ def _check_index(
                 logger.debug(
                     "Column dtypes identified: {}".format(details["index_dtypes"])
                 )
-
+                logger.debug(df.head())
                 # Drop empty rows
                 df = (
                     df.dropna(axis=0, how="all")

src/otoole/preprocess/datapackage.json

@@ -294,7 +294,7 @@
                     },
                     {
                         "name": "VALUE",
-                        "type": "string",
+                        "type": "number",
                         "format": "default"
                     }
                 ],
@@ -1354,7 +1354,7 @@
                     },
                     {
                         "name": "VALUE",
-                        "type": "string",
+                        "type": "number",
                         "format": "default"
                     }
                 ],
@@ -1616,7 +1616,7 @@
                     },
                     {
                         "name": "VALUE",
-                        "type": "string",
+                        "type": "number",
                         "format": "default"
                     }
                 ],
@@ -1860,7 +1860,7 @@
                     },
                     {
                         "name": "VALUE",
-                        "type": "string",
+                        "type": "number",
                         "format": "default"
                     }
                 ],
@@ -1926,7 +1926,7 @@
                     },
                     {
                         "name": "VALUE",
-                        "type": "string",
+                        "type": "number",
                         "format": "default"
                     }
                 ],
@@ -2206,7 +2206,7 @@
                     },
                     {
                         "name": "VALUE",
-                        "type": "string",
+                        "type": "number",
                         "format": "default"
                     }
                 ],
@@ -2315,7 +2315,7 @@
                     },
                     {
                         "name": "VALUE",
-                        "type": "string",
+                        "type": "number",
                         "format": "default"
                     }
                 ],
@@ -2493,7 +2493,7 @@
                     },
                     {
                         "name": "VALUE",
-                        "type": "string",
+                        "type": "number",
                         "format": "default"
                     }
                 ],

src/otoole/preprocess/longify_data.py

@@ -1,9 +1,9 @@
 """Read in a folder of irregular wide-format csv files and write them out as narrow csvs
 """
 import logging
-from typing import Dict
-
+import numpy as np
 import pandas as pd
+from typing import Dict
 
 logger = logging.getLogger()
 
@@ -30,30 +30,32 @@ def check_set_datatype(
 
 
 def check_datatypes(
-    narrow: pd.DataFrame, config_details: Dict, parameter: str
+    df: pd.DataFrame, config_details: Dict, parameter: str
 ) -> pd.DataFrame:
     """Checks a parameters datatypes
 
     Arguments
     ---------
-    narrow : pandas.DataFrame
+    df : pandas.DataFrame
         The parameter data
     config_details : dict
         The configuration dictionary
     parameter : str
         The name of the parameter
     """
     logger.info("Checking datatypes for %s", parameter)
+    logger.debug(df.columns)
     dtypes = {}
 
-    for column in narrow.columns:
+    for column in df.columns:
         if column == "VALUE":
             datatype = config_details[parameter]["dtype"]
             dtypes["VALUE"] = datatype
         else:
             datatype = config_details[column]["dtype"]
             dtypes[column] = datatype
-        if narrow[column].dtype != datatype:
+            logger.debug(f"Found {datatype} for column {column}")
+        if df[column].dtype != datatype:
             logger.info(
                 "dtype of column %s does not match %s for parameter %s",
                 column,
@@ -63,14 +65,15 @@ def check_datatypes(
             if datatype == "int":
                 dtypes[column] = "int64"
                 try:
-                    narrow[column] = narrow[column].apply(_cast_to_int)
+                    df[column] = df[column].apply(_cast_to_int)
                 except ValueError as ex:
                     msg = "Unable to apply datatype for column {}: {}".format(
                         column, str(ex)
                     )
                     raise ValueError(msg)
-    return narrow.astype(dtypes)
+
+    return df.astype(dtypes)
 
 
 def _cast_to_int(value):
-    return int(float(value))
+    return np.int64(float(value))

src/otoole/read_strategies.py

@@ -9,6 +9,7 @@
 
 from otoole.input import ReadStrategy
 from otoole.preprocess.longify_data import check_datatypes, check_set_datatype
+from otoole.utils import read_datapackage_schema_into_config
 
 logger = logging.getLogger(__name__)
 
@@ -66,7 +67,6 @@ def _check_parameter(self, df: pd.DataFrame, expected_headers: List, name: str):
         logger.debug("Expected headers for %s: %s", name, expected_headers)
 
         if "REGION" in expected_headers and "REGION" not in actual_headers:
-            logger.error("No 'REGION' column provided for %s", name)
             raise ValueError("No REGION column provided for %s", name)
 
         if "MODEOFOPERATION" in actual_headers:
@@ -198,6 +198,9 @@ def read(
         inputs = read_datapackage(filepath)
         default_resource = inputs.pop("default_values").set_index("name").to_dict()
         default_values = default_resource["default_value"]
+        self.input_config = read_datapackage_schema_into_config(
+            filepath, default_values
+        )
         inputs = self._check_index(inputs)
         return inputs, default_values
 

src/otoole/results/config.yaml

@@ -172,3 +172,6 @@ UseByTechnology:
     dtype: float
     default: 0
     calculated: False
+_REGION:
+    type: set
+    dtype: str