From a3a441ed1398cec231ccddde1fd87f76fe8f626f Mon Sep 17 00:00:00 2001 From: jessicasyu <15913767+jessicasyu@users.noreply.github.com> Date: Tue, 27 Feb 2024 11:25:43 -0500 Subject: [PATCH 1/6] Update convert model units to estimate ds and dt --- .../output/convert_model_units.py | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/src/arcade_collection/output/convert_model_units.py b/src/arcade_collection/output/convert_model_units.py index c3c398b..edad016 100644 --- a/src/arcade_collection/output/convert_model_units.py +++ b/src/arcade_collection/output/convert_model_units.py @@ -1,11 +1,21 @@ +import re from typing import Optional, Union import pandas as pd def convert_model_units( - data: pd.DataFrame, ds: float, dt: float, regions: Optional[Union[list[str], str]] = None + data: pd.DataFrame, + ds: Optional[float], + dt: Optional[float], + regions: Optional[Union[list[str], str]] = None, ) -> None: + if dt is None: + dt = data["KEY"].apply(estimate_temporal_resolution) + + if ds is None: + ds = data["KEY"].apply(estimate_spatial_resolution) + data["time"] = round(dt * data["TICK"], 2) if "NUM_VOXELS" in data.columns: @@ -35,3 +45,21 @@ def convert_model_units( data[f"volume.{region}"] = ds * ds * ds * data[f"NUM_VOXELS.{region}"] data[f"height.{region}"] = ds * (data[f"MAX_Z.{region}"] - data[f"MIN_Z.{region}"]) + + +def estimate_temporal_resolution(key: str) -> float: + match = re.findall(r"[_]?DT([0-9]+)[_]?", key) + + if len(match) == 1: + return float(match[0]) / 60 + + return 1.0 + + +def estimate_spatial_resolution(key: str) -> float: + match = re.findall(r"[_]?DS([0-9]+)[_]?", key) + + if len(match) == 1: + return float(match[0]) + + return 1.0 From 78de359910c4f5bf150563a430fee283605055b0 Mon Sep 17 00:00:00 2001 From: jessicasyu <15913767+jessicasyu@users.noreply.github.com> Date: Tue, 27 Feb 2024 11:28:21 -0500 Subject: [PATCH 2/6] Add docstrings to convert model units task --- .../output/convert_model_units.py | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/src/arcade_collection/output/convert_model_units.py b/src/arcade_collection/output/convert_model_units.py index edad016..d7baed4 100644 --- a/src/arcade_collection/output/convert_model_units.py +++ b/src/arcade_collection/output/convert_model_units.py @@ -10,6 +10,48 @@ def convert_model_units( dt: Optional[float], regions: Optional[Union[list[str], str]] = None, ) -> None: + """ + Converts data from simulation units to true units. + + Simulations use spatial unit of voxels and temporal unit of ticks. Spatial + resolution (microns/voxel) and temporal resolution (hours/tick) are used to + convert data to true units. If spatial or temporal resolution is not given, + they will be estimated from the ``KEY`` column of the data. + + The following columns are added to the data: + + ============= =================== ============================= + Target column Source column(s) Calculation + ============= =================== ============================= + ``time`` ``TICK`` ``dt * TICK`` + ``volume`` ``NUM_VOXELS`` ``ds * ds * ds * NUM_VOXELS`` + ``height`` ``MAX_Z`` ``MIN_Z`` ``ds * (MAX_Z - MIN_Z + 1)`` + ``cx`` ``CX`` ``ds * CX`` + ``cy`` ``CY`` ``ds * CY`` + ``cz`` ``CZ`` ``ds * CZ`` + ============= =================== ============================= + + For each region (other than ``DEFAULT``), volume and height are calculated: + + ================= ================================= ========================================== + Target column Source column(s) Calculation + ================= ================================= ========================================== + ``volume.REGION`` ``NUM_VOXELS.REGION`` ``ds * ds * ds * NUM_VOXELS.REGION`` + ``height.REGION`` ``MAX_Z.REGION`` ``MIN_Z.REGION`` ``ds * (MAX_Z.REGION - MIN_Z.REGION + 1)`` + ================= ================================= ========================================== + + Parameters + ---------- + data + Parsed simulation data. + ds + Spatial resolution in microns/voxel, use None to estimate from keys. + dt + Temporal resolution in hours/tick, use None to estimate from keys. + regions + List of regions. + """ + if dt is None: dt = data["KEY"].apply(estimate_temporal_resolution) @@ -48,6 +90,24 @@ def convert_model_units( def estimate_temporal_resolution(key: str) -> float: + """ + Estimates temporal resolution based on condition key. + + If the key contains ``DT##``, where ``##`` denotes the temporal resolution + in minutes/tick, temporal resolution is estimated from ``##``. Otherwise, + the default temporal resolution is 1 hours/tick. + + Parameters + ---------- + key + Condition key. + + Returns + ------- + : + Temporal resolution (hours/tick). + """ + match = re.findall(r"[_]?DT([0-9]+)[_]?", key) if len(match) == 1: @@ -57,6 +117,24 @@ def estimate_temporal_resolution(key: str) -> float: def estimate_spatial_resolution(key: str) -> float: + """ + Estimates spatial resolution based on condition key. + + If the key contains ``DS##``, where ``##`` denotes the spatial resolution + in micron/voxel, spatial resolution is estimated from ``##``. Otherwise, + the default spatial resolution is 1 micron/voxel. + + Parameters + ---------- + key + Condition key. + + Returns + ------- + : + Spatial resolution (micron/voxel). + """ + match = re.findall(r"[_]?DS([0-9]+)[_]?", key) if len(match) == 1: From 3d49251d19071180bfd4b2ac1e930a91df8fbb9c Mon Sep 17 00:00:00 2001 From: jessicasyu <15913767+jessicasyu@users.noreply.github.com> Date: Wed, 28 Feb 2024 17:33:18 -0500 Subject: [PATCH 3/6] Add tests for convert model units --- .../output/convert_model_units.py | 16 ++---- .../output/test_convert_model_units.py | 53 +++++++++++++++++++ 2 files changed, 57 insertions(+), 12 deletions(-) create mode 100644 tests/arcade_collection/output/test_convert_model_units.py diff --git a/src/arcade_collection/output/convert_model_units.py b/src/arcade_collection/output/convert_model_units.py index d7baed4..9c28bf4 100644 --- a/src/arcade_collection/output/convert_model_units.py +++ b/src/arcade_collection/output/convert_model_units.py @@ -108,12 +108,8 @@ def estimate_temporal_resolution(key: str) -> float: Temporal resolution (hours/tick). """ - match = re.findall(r"[_]?DT([0-9]+)[_]?", key) - - if len(match) == 1: - return float(match[0]) / 60 - - return 1.0 + matches = [re.fullmatch(r"DT([0-9]+)", k) for k in key.split("_")] + return next((float(match.group(1)) / 60 for match in matches if match is not None), 1.0) def estimate_spatial_resolution(key: str) -> float: @@ -135,9 +131,5 @@ def estimate_spatial_resolution(key: str) -> float: Spatial resolution (micron/voxel). """ - match = re.findall(r"[_]?DS([0-9]+)[_]?", key) - - if len(match) == 1: - return float(match[0]) - - return 1.0 + matches = [re.fullmatch(r"DS([0-9]+)", k) for k in key.split("_")] + return next((float(match.group(1)) for match in matches if match is not None), 1.0) diff --git a/tests/arcade_collection/output/test_convert_model_units.py b/tests/arcade_collection/output/test_convert_model_units.py new file mode 100644 index 0000000..465c59c --- /dev/null +++ b/tests/arcade_collection/output/test_convert_model_units.py @@ -0,0 +1,53 @@ +import random +import unittest + +from arcade_collection.output.convert_model_units import ( + estimate_spatial_resolution, + estimate_temporal_resolution, +) + + +class TestExtractVoxelContours(unittest.TestCase): + def test_estimate_temporal_resolution_missing_temporal_key(self): + self.assertEqual(1, estimate_temporal_resolution("")) + self.assertEqual(1, estimate_temporal_resolution("A")) + self.assertEqual(1, estimate_temporal_resolution("A_B")) + self.assertEqual(1, estimate_temporal_resolution("A_B_C")) + + def test_estimate_temporal_resolution_valid_temporal_key(self): + dt = int(random.random() * 10) + dt_key = f"DT{dt:03d}" + + self.assertEqual(dt / 60, estimate_temporal_resolution(f"{dt_key}_B_C")) + self.assertEqual(dt / 60, estimate_temporal_resolution(f"A_{dt_key}_C")) + self.assertEqual(dt / 60, estimate_temporal_resolution(f"A_B_{dt_key}")) + + def test_estimate_temporal_resolution_invalid_temporal_key(self): + dt = int(random.random() * 10) + dt_key = f"DT{dt:03d}x" + + self.assertEqual(1, estimate_temporal_resolution(f"{dt_key}_B_C")) + self.assertEqual(1, estimate_temporal_resolution(f"A_{dt_key}_C")) + self.assertEqual(1, estimate_temporal_resolution(f"A_B_{dt_key}")) + + def test_estimate_spatial_resolution_missing_spatial_key(self): + self.assertEqual(1, estimate_spatial_resolution("")) + self.assertEqual(1, estimate_spatial_resolution("A")) + self.assertEqual(1, estimate_spatial_resolution("A_B")) + self.assertEqual(1, estimate_spatial_resolution("A_B_C")) + + def test_estimate_spatial_resolution_valid_spatial_key(self): + ds = int(random.random() * 10) + ds_key = f"DS{ds:03d}" + + self.assertEqual(ds, estimate_spatial_resolution(f"{ds_key}_B_C")) + self.assertEqual(ds, estimate_spatial_resolution(f"A_{ds_key}_C")) + self.assertEqual(ds, estimate_spatial_resolution(f"A_B_{ds_key}")) + + def test_estimate_spatial_resolution_invalid_spatiall_key(self): + ds = int(random.random() * 10) + ds_key = f"DS{ds:03d}x" + + self.assertEqual(1, estimate_spatial_resolution(f"{ds_key}_B_C")) + self.assertEqual(1, estimate_spatial_resolution(f"A_{ds_key}_C")) + self.assertEqual(1, estimate_spatial_resolution(f"A_B_{ds_key}")) From ca70b0249551af32038ab4a00695c89e9792219b Mon Sep 17 00:00:00 2001 From: jessicasyu <15913767+jessicasyu@users.noreply.github.com> Date: Thu, 29 Feb 2024 10:29:42 -0500 Subject: [PATCH 4/6] Move conversions to separate methods --- .../output/convert_model_units.py | 132 +++++++++++++++--- 1 file changed, 112 insertions(+), 20 deletions(-) diff --git a/src/arcade_collection/output/convert_model_units.py b/src/arcade_collection/output/convert_model_units.py index 9c28bf4..69db917 100644 --- a/src/arcade_collection/output/convert_model_units.py +++ b/src/arcade_collection/output/convert_model_units.py @@ -21,7 +21,7 @@ def convert_model_units( The following columns are added to the data: ============= =================== ============================= - Target column Source column(s) Calculation + Target column Source column(s) Conversion ============= =================== ============================= ``time`` ``TICK`` ``dt * TICK`` ``volume`` ``NUM_VOXELS`` ``ds * ds * ds * NUM_VOXELS`` @@ -31,19 +31,30 @@ def convert_model_units( ``cz`` ``CZ`` ``ds * CZ`` ============= =================== ============================= - For each region (other than ``DEFAULT``), volume and height are calculated: + For each region (other than ``DEFAULT``), the following columns are added to the data: ================= ================================= ========================================== - Target column Source column(s) Calculation + Target column Source column(s) Conversion ================= ================================= ========================================== ``volume.REGION`` ``NUM_VOXELS.REGION`` ``ds * ds * ds * NUM_VOXELS.REGION`` ``height.REGION`` ``MAX_Z.REGION`` ``MIN_Z.REGION`` ``ds * (MAX_Z.REGION - MIN_Z.REGION + 1)`` ================= ================================= ========================================== + The following property columns are rescaled: + + ===================== ===================== ========================== + Target column Source column(s) Conversion + ===================== ===================== ========================== + ``area`` ``area`` ``ds * ds * area`` + ``perimeter`` ``perimeter`` ``ds * perimeter`` + ``axis_major_length`` ``axis_major_length`` ``ds * axis_major_length`` + ``axis_minor_length`` ``axis_minor_length`` ``ds * axis_minor_length`` + ===================== ===================== ========================== + Parameters ---------- data - Parsed simulation data. + Simulation data. ds Spatial resolution in microns/voxel, use None to estimate from keys. dt @@ -58,35 +69,116 @@ def convert_model_units( if ds is None: ds = data["KEY"].apply(estimate_spatial_resolution) - data["time"] = round(dt * data["TICK"], 2) + convert_temporal_units(data, dt) + convert_spatial_units(data, ds) + + if regions is None: + return + + if isinstance(regions, str): + regions = [regions] + + for region in regions: + if region == "DEFAULT": + continue + + convert_spatial_units(data, ds, region) + + +def convert_temporal_units(data: pd.DataFrame, dt: float) -> None: + """ + Converts temporal data from simulation units to true units. + + Simulations use temporal unit of ticks. Temporal resolution (hours/tick) is + used to convert data to true units. + + The following temporal columns are converted: + + ============= =================== ============================= + Target column Source column(s) Conversion + ============= =================== ============================= + ``time`` ``TICK`` ``dt * TICK`` + ============= =================== ============================= + + Parameters + ---------- + data + Simulation data. + dt + Temporal resolution in hours/tick. + """ + + if "TICK" in data.columns: + data["time"] = round(dt * data["TICK"], 2) - if "NUM_VOXELS" in data.columns: - data["volume"] = ds * ds * ds * data["NUM_VOXELS"] - if "MAX_Z" in data.columns and "MIN_Z" in data.columns: - data["height"] = ds * (data["MAX_Z"] - data["MIN_Z"] + 1) +def convert_spatial_units(data: pd.DataFrame, ds: float, region: Optional[str] = None) -> None: + """ + Converts spatial data from simulation units to true units. + + Simulations use spatial unit of voxels. Spatial resolution (microns/voxel) + is used to convert data to true units. + + The following spatial columns are converted: + + ===================== ===================== ============================= + Target column Source column(s) Conversion + ===================== ===================== ============================= + ``volume`` ``NUM_VOXELS`` ``ds * ds * ds * NUM_VOXELS`` + ``height`` ``MAX_Z`` ``MIN_Z`` ``ds * (MAX_Z - MIN_Z + 1)`` + ``cx`` ``CX`` ``ds * CX`` + ``cy`` ``CY`` ``ds * CY`` + ``cz`` ``CZ`` ``ds * CZ`` + ``area`` ``area`` ``ds * ds * area`` + ``perimeter`` ``perimeter`` ``ds * perimeter`` + ``axis_major_length`` ``axis_major_length`` ``ds * axis_major_length`` + ``axis_minor_length`` ``axis_minor_length`` ``ds * axis_minor_length`` + ===================== ===================== ============================= + + Note that the centroid columns (``cx``, ``cy``, and ``cz``) are only + converted for the entire cell (``region == None``). - if "CX" in data.columns: + Parameters + ---------- + data + Simulation data. + ds + Spatial resolution in microns/voxel. + region + Name of region. + """ + + suffix = "" if region is None else f".{region}" + + if f"NUM_VOXELS{suffix}" in data.columns: + data[f"volume{suffix}"] = ds * ds * ds * data[f"NUM_VOXELS{suffix}"] + + if f"MAX_Z{suffix}" in data.columns and f"MIN_Z{suffix}" in data.columns: + data[f"height{suffix}"] = ds * (data[f"MAX_Z{suffix}"] - data[f"MIN_Z{suffix}"] + 1) + + if "CX" in data.columns and region is None: data["cx"] = ds * data["CX"] - if "CY" in data.columns: + if "CY" in data.columns and region is None: data["cy"] = ds * data["CY"] - if "CZ" in data.columns: + if "CZ" in data.columns and region is None: data["cz"] = ds * data["CZ"] - if regions is None: - return + property_conversions = [ + ("area", ds * ds), + ("perimeter", ds), + ("axis_major_length", ds), + ("axis_minor_length", ds), + ] - if isinstance(regions, str): - regions = [regions] + for name, conversion in property_conversions: + column = f"{name}{suffix}" - for region in regions: - if region == "DEFAULT": + if column not in data.columns: continue - data[f"volume.{region}"] = ds * ds * ds * data[f"NUM_VOXELS.{region}"] - data[f"height.{region}"] = ds * (data[f"MAX_Z.{region}"] - data[f"MIN_Z.{region}"]) + data[column] = data[column] * conversion def estimate_temporal_resolution(key: str) -> float: From 664d0e4e0e014aead37cd977c56752b0c4b5e611 Mon Sep 17 00:00:00 2001 From: jessicasyu <15913767+jessicasyu@users.noreply.github.com> Date: Thu, 29 Feb 2024 12:27:46 -0500 Subject: [PATCH 5/6] Clean up docstring table --- src/arcade_collection/output/convert_model_units.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/arcade_collection/output/convert_model_units.py b/src/arcade_collection/output/convert_model_units.py index 69db917..e10ef3a 100644 --- a/src/arcade_collection/output/convert_model_units.py +++ b/src/arcade_collection/output/convert_model_units.py @@ -94,11 +94,11 @@ def convert_temporal_units(data: pd.DataFrame, dt: float) -> None: The following temporal columns are converted: - ============= =================== ============================= - Target column Source column(s) Conversion - ============= =================== ============================= - ``time`` ``TICK`` ``dt * TICK`` - ============= =================== ============================= + ============= ================ ============= + Target column Source column(s) Conversion + ============= ================ ============= + ``time`` ``TICK`` ``dt * TICK`` + ============= ================ ============= Parameters ---------- From bf94095a246cba7fece046b71c8c78d4e1f6b11e Mon Sep 17 00:00:00 2001 From: jessicasyu <15913767+jessicasyu@users.noreply.github.com> Date: Thu, 29 Feb 2024 20:57:16 -0500 Subject: [PATCH 6/6] Fix centroid conversions --- .../output/convert_model_units.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/arcade_collection/output/convert_model_units.py b/src/arcade_collection/output/convert_model_units.py index e10ef3a..590ec28 100644 --- a/src/arcade_collection/output/convert_model_units.py +++ b/src/arcade_collection/output/convert_model_units.py @@ -26,9 +26,9 @@ def convert_model_units( ``time`` ``TICK`` ``dt * TICK`` ``volume`` ``NUM_VOXELS`` ``ds * ds * ds * NUM_VOXELS`` ``height`` ``MAX_Z`` ``MIN_Z`` ``ds * (MAX_Z - MIN_Z + 1)`` - ``cx`` ``CX`` ``ds * CX`` - ``cy`` ``CY`` ``ds * CY`` - ``cz`` ``CZ`` ``ds * CZ`` + ``cx`` ``CENTER_X`` ``ds * CENTER_X`` + ``cy`` ``CENTER_Y`` ``ds * CENTER_Y`` + ``cz`` ``CENTER_Z`` ``ds * CENTER_Z`` ============= =================== ============================= For each region (other than ``DEFAULT``), the following columns are added to the data: @@ -126,9 +126,9 @@ def convert_spatial_units(data: pd.DataFrame, ds: float, region: Optional[str] = ===================== ===================== ============================= ``volume`` ``NUM_VOXELS`` ``ds * ds * ds * NUM_VOXELS`` ``height`` ``MAX_Z`` ``MIN_Z`` ``ds * (MAX_Z - MIN_Z + 1)`` - ``cx`` ``CX`` ``ds * CX`` - ``cy`` ``CY`` ``ds * CY`` - ``cz`` ``CZ`` ``ds * CZ`` + ``cx`` ``CENTER_X`` ``ds * CENTER_X`` + ``cy`` ``CENTER_Y`` ``ds * CENTER_Y`` + ``cz`` ``CENTER_Z`` ``ds * CENTER_Z`` ``area`` ``area`` ``ds * ds * area`` ``perimeter`` ``perimeter`` ``ds * perimeter`` ``axis_major_length`` ``axis_major_length`` ``ds * axis_major_length`` @@ -156,14 +156,14 @@ def convert_spatial_units(data: pd.DataFrame, ds: float, region: Optional[str] = if f"MAX_Z{suffix}" in data.columns and f"MIN_Z{suffix}" in data.columns: data[f"height{suffix}"] = ds * (data[f"MAX_Z{suffix}"] - data[f"MIN_Z{suffix}"] + 1) - if "CX" in data.columns and region is None: - data["cx"] = ds * data["CX"] + if "CENTER_X" in data.columns and region is None: + data["cx"] = ds * data["CENTER_X"] - if "CY" in data.columns and region is None: - data["cy"] = ds * data["CY"] + if "CENTER_Y" in data.columns and region is None: + data["cy"] = ds * data["CENTER_Y"] - if "CZ" in data.columns and region is None: - data["cz"] = ds * data["CZ"] + if "CENTER_Z" in data.columns and region is None: + data["cz"] = ds * data["CENTER_Z"] property_conversions = [ ("area", ds * ds),