469 update retrofit dataset

.gitignore

@@ -51,6 +51,7 @@ original/
 *.json
 !tests/data/*/*.json
 !tests/data/*.json
+!tests/data/retrofit/*.csv
 
 # Seaside ipopt constants
 pyincore/analyses/seasidecge/solverconstants/ipopt_cons.py

pyincore/dfr3service.py

@@ -8,6 +8,8 @@
 import re
 from urllib.parse import urljoin
 from typing import Dict
+import math
+import scipy
 
 import pyincore.globals as pyglobals
 from pyincore.decorators import forbid_offline
@@ -56,7 +58,7 @@ def __init__(self):
 
 
 class MappingResponse(object):
-    def __init__(self, sets: Dict[str, any]=dict(), mapping: Dict[str, str]=dict()):
+    def __init__(self, sets: Dict[str, any] = dict(), mapping: Dict[str, str] = dict()):
         self.sets = sets
         self.mapping = mapping
 
@@ -175,23 +177,32 @@ def create_dfr3_set(self, dfr3_set: dict, timeout=(30, 600), **kwargs):
         r = self.client.post(url, json=dfr3_set, timeout=timeout, **kwargs)
         return return_http_response(r).json()
 
-    def match_inventory(self, mapping: MappingSet, inventories: list, entry_key: str, add_info: list = None):
+    def match_inventory(self, mapping: MappingSet, inventories: list, entry_key: str = None, add_info: list = None):
         """This method is intended to replace the match_inventory method in the future. The functionality is same as
         match_inventory but instead of dfr3_sets in plain json, dfr3 curves will be represented in
         FragilityCurveSet Object.
 
         Args:
             mapping (obj): MappingSet Object that has the rules and entries.
-            inventories (list): A list of inventories. Each item is a casted fiona object
-            entry_key (str): keys such as PGA, pgd, and etc.
-            add_info (None, dict): additional information that used to match rules, e.g. retrofit strategy per building.
+            inventories (list): A list of inventories. Each item is a fiona object
+            entry_key (None, str): Mapping Entry Key e.g. Non-retrofit Fragility ID Code, retrofit_method_1, etc.
+            add_info (None, list): additional information that used to match rules, e.g. retrofit strategy per building.
 
         Returns:
              dict: A dictionary of {"inventory id": FragilityCurveSet object}.
 
         """
         dfr3_sets = {}
 
+        # find default mapping entry key if not provided
+        if entry_key is None:
+            for m in mapping.mappingEntryKeys:
+                if "defaultKey" in m and m["defaultKey"] is True:
+                    entry_key = m["name"]
+                    break
+        if entry_key is None:
+            raise ValueError("Entry key not provided and no default entry key found in the mapping!")
+
         # loop through inventory to match the rules
         matched_curve_ids = []
         for inventory in inventories:
@@ -202,21 +213,57 @@ def match_inventory(self, mapping: MappingSet, inventories: list, entry_key: str
                     inventory["properties"]["efacility"] is None:
                 inventory["properties"]["efacility"] = ""
 
-            # if additional information presented, merge inventory properties with that additional information
+            # if additional information e.g. Retrofit presented, merge inventory properties with that additional
+            # information
             if add_info is not None:
                 for add_info_row in add_info:
+                    # assume no duplicated guid
                     if inventory["properties"].get("guid") is not None and \
                             add_info_row.get("guid") is not None and \
                             inventory["properties"].get("guid") == add_info_row.get("guid"):
+                        # merging { "retrofit_key":xxx, "retrofit_value": yyy }
                         inventory["properties"].update(add_info_row)
-                        break  # assume no duplicated guid
+
+                        # For retrofit: if targetColumn and expression exist, building inventory properties will be
+                        # updated
+                        target_column = None
+                        expression = None
+                        type = None
+                        for m in mapping.mappingEntryKeys:
+                            if m["name"] == add_info_row["retrofit_key"]:
+                                target_column = m["config"]["targetColumn"] if ("config" in m and "targetColumn" in
+                                                                                m["config"]) else None
+                                expression = m["config"]["expression"] if ("config" in m and "expression" in m[
+                                    "config"]) else None
+                                type = m["config"]["type"] if ("config" in m and "type" in m["config"]) else None
+                        if target_column is not None and expression is not None:
+                            if target_column in inventory["properties"].keys():
+                                retrofit_value = add_info_row["retrofit_value"]
+                                if type and type == "number":
+                                    retrofit_value = float(retrofit_value)
+
+                                # Dangerous!
+                                exec(f"inventory['properties'][target_column]{expression}")
+
+                            else:
+                                raise ValueError("targetColumn: " + target_column + " not found in inventory "
+                                                                                    "properties!")
+                        break
+
+            # if retrofit key exist, use retrofit key otherwise use default key
+            retrofit_entry_key = None
+            if "retrofit_key" in inventory["properties"]:
+                retrofit_entry_key = inventory["properties"]["retrofit_key"]
 
             for m in mapping.mappings:
                 # for old format rule matching [[]]
                 # [[ and ] or [ and ]]
                 if isinstance(m.rules, list):
                     if self._property_match_legacy(rules=m.rules, properties=inventory["properties"]):
-                        curve = m.entry[entry_key]
+                        if retrofit_entry_key is not None and retrofit_entry_key in m.entry.keys():
+                            curve = m.entry[retrofit_entry_key]
+                        else:
+                            curve = m.entry[entry_key]
                         dfr3_sets[inventory['id']] = curve
 
                         # if it's string:id; then need to fetch it from remote and cast to fragility3curve object
@@ -230,10 +277,13 @@ def match_inventory(self, mapping: MappingSet, inventories: list, entry_key: str
                 # {"AND": [xx, "OR": [yy, yy], "AND": {"OR":["zz", "zz"]]}
                 elif isinstance(m.rules, dict):
                     if self._property_match(rules=m.rules, properties=inventory["properties"]):
-                        curve = m.entry[entry_key]
+                        if retrofit_entry_key is not None and retrofit_entry_key in m.entry.keys():
+                            curve = m.entry[retrofit_entry_key]
+                        else:
+                            curve = m.entry[entry_key]
                         dfr3_sets[inventory['id']] = curve
 
-                        # if it's string:id; then need to fetch it from remote and cast to fragility3curve object
+                        # if it's string:id; then need to fetch it from remote and cast to dfr3 curve object
                         if isinstance(curve, str) and curve not in matched_curve_ids:
                             matched_curve_ids.append(curve)
 
@@ -255,21 +305,30 @@ def match_inventory(self, mapping: MappingSet, inventories: list, entry_key: str
 
         return dfr3_sets
 
-    def match_list_of_dicts(self, mapping: MappingSet, inventories: list, entry_key: str):
+    def match_list_of_dicts(self, mapping: MappingSet, inventories: list, entry_key: str = None):
         """This method is same as match_inventory, except it takes a simple list of dictionaries that contains the items
         to be mapped in the rules. The match_inventory method takes a list of fiona objects
 
         Args:
             mapping (obj): MappingSet Object that has the rules and entries.
             inventories (list): A list of inventories. Each item of the list is a simple dictionary
-            entry_key (str): keys such as PGA, pgd, and etc.
+            entry_key (None, str): Mapping Entry Key e.g. Non-retrofit Fragility ID Code, retrofit_method_1, etc.
 
         Returns:
              dict: A dictionary of {"inventory id": FragilityCurveSet object}.
 
         """
         dfr3_sets = {}
 
+        # find default mapping entry key if not provided
+        if entry_key is None:
+            for m in mapping.mappingEntryKeys:
+                if "defaultKey" in m and m["defaultKey"] is True:
+                    entry_key = m["name"]
+                    break
+        if entry_key is None:
+            raise ValueError("Entry key not provided and no default entry key found in the mapping!")
+
         # loop through inventory to match the rules
         matched_curve_ids = []
         for inventory in inventories:
@@ -446,7 +505,8 @@ def extract_inventory_class_legacy(rules):
         """This method will extract the inventory class name from a mapping rule. E.g. PWT2/PPP1
 
         Args:
-            rules (list): The outer list is applying "OR" rule and the inner list is applying an "AND" rule. e.g. list(["java.lang.String utilfcltyc EQUALS 'PWT2'"],["java.lang.String utilfcltyc EQUALS 'PPP1'"])
+            rules (list): The outer list is applying "OR" rule and the inner list is applying an "AND" rule.
+            e.g. list(["java.lang.String utilfcltyc EQUALS 'PWT2'"],["java.lang.String utilfcltyc EQUALS 'PPP1'"])
 
         Returns:
             inventory_class (str): extracted inventory class name. "/" stands for or and "+" stands for and
@@ -471,7 +531,8 @@ def extract_inventory_class(rules):
         """This method will extract the inventory class name from a mapping rule. E.g. PWT2/PPP1
 
         Args:
-            rules (dict): e.g. { "AND": ["java.lang.String utilfcltyc EQUALS 'PWT2'", "java.lang.String utilfcltyc EQUALS 'PPP1'"]}
+            rules (dict): e.g. { "AND": ["java.lang.String utilfcltyc EQUALS 'PWT2'",
+            "java.lang.String utilfcltyc EQUALS 'PPP1'"]}
 
         Returns:
             inventory_class (str): extracted inventory class name. "/" stands for or and "+" stands for and

pyincore/models/mappingset.py

@@ -18,15 +18,17 @@ class MappingSet:
     """
 
     def __init__(self, metadata):
-        self.id = metadata["id"]
-        self.name = metadata["name"]
-        self.hazard_type = metadata["hazardType"]
-        self.inventory_type = metadata['inventoryType']
 
-        if 'dataType' in metadata:
-            self.data_type = metadata["dataType"]
+        self.id = metadata["id"] if "id" in metadata else ""
+        self.name = metadata["name"] if "name" in metadata else ""
+        self.hazard_type = metadata["hazardType"] if "hazardType" in metadata else ""
+        self.inventory_type = metadata['inventoryType'] if "inventoryType" in metadata else ""
+        if "mappingEntryKeys" in metadata and metadata["mappingEntryKeys"] is not None:
+            self.mappingEntryKeys = metadata["mappingEntryKeys"]
         else:
-            self.data_type = "incore:dfr3MappingSet"
+            self.mappingEntryKeys = []
+
+        self.data_type = metadata["dataType"] if "dataType" in metadata else "incore:dfr3MappingSet"
 
         mappings = []
         for m in metadata['mappings']:
@@ -54,7 +56,7 @@ def from_json_str(cls, json_str):
         return cls(json.loads(json_str))
 
     @classmethod
-    def from_json_file(cls, file_path, data_type):
+    def from_json_file(cls, file_path, data_type="incore:dfr3MappingSet"):
         """Get dfr3 mapping from the file.
 
         Args: