From ce98283f212a061f31863a4018a50cb465a498e2 Mon Sep 17 00:00:00 2001 From: atmorling Date: Wed, 4 Dec 2024 15:03:55 +0200 Subject: [PATCH] Fix get_events time parse (#353) --- ecoscope/io/earthranger.py | 4 ++-- ecoscope/io/earthranger_utils.py | 5 +++-- tests/test_earthranger_io.py | 18 ++++++++++++++++++ 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/ecoscope/io/earthranger.py b/ecoscope/io/earthranger.py index 90364c71..9b935e4a 100644 --- a/ecoscope/io/earthranger.py +++ b/ecoscope/io/earthranger.py @@ -626,8 +626,8 @@ def get_events( ) gdf = gpd.GeoDataFrame(df) - if not df.empty: - df = clean_time_cols(df) + if not gdf.empty: + gdf = clean_time_cols(gdf) if gdf.loc[0, "location"] is not None: gdf.loc[~gdf["geojson"].isna(), "geometry"] = gpd.GeoDataFrame.from_features( gdf.loc[~gdf["geojson"].isna(), "geojson"] diff --git a/ecoscope/io/earthranger_utils.py b/ecoscope/io/earthranger_utils.py index 0d5c6924..f622a316 100644 --- a/ecoscope/io/earthranger_utils.py +++ b/ecoscope/io/earthranger_utils.py @@ -3,6 +3,8 @@ import pandas as pd from dateutil import parser +TIME_COLS = ["time", "created_at", "updated_at", "end_time", "last_position_date", "recorded_at", "fixtime"] + def clean_kwargs(addl_kwargs={}, **kwargs): for k in addl_kwargs.keys(): @@ -38,8 +40,7 @@ def to_gdf(df): def clean_time_cols(df): - time_cols = ["time", "created_at", "updated_at", "end_time", "last_position_date", "recorded_at", "fixtime"] - for col in time_cols: + for col in TIME_COLS: if col in df.columns and not pd.api.types.is_datetime64_ns_dtype(df[col]): # convert x is not None to pd.isna(x) is False df[col] = df[col].apply(lambda x: pd.to_datetime(parser.parse(x), utc=True) if not pd.isna(x) else None) diff --git a/tests/test_earthranger_io.py b/tests/test_earthranger_io.py index f45c62de..6f6d7dfc 100644 --- a/tests/test_earthranger_io.py +++ b/tests/test_earthranger_io.py @@ -11,9 +11,17 @@ import ecoscope from erclient import ERClientException +from ecoscope.io.earthranger_utils import TIME_COLS + pytestmark = pytest.mark.io +def check_time_is_parsed(df): + for col in TIME_COLS: + if col in df.columns: + assert pd.api.types.is_datetime64_ns_dtype(df[col]) or df[col].isna().all() + + def test_get_subject_observations(er_io): relocations = er_io.get_subject_observations( subject_ids=er_io.SUBJECT_IDS, @@ -26,6 +34,7 @@ def test_get_subject_observations(er_io): assert "groupby_col" in relocations assert "fixtime" in relocations assert "extra__source" in relocations + check_time_is_parsed(relocations) def test_get_source_observations(er_io): @@ -36,6 +45,7 @@ def test_get_source_observations(er_io): assert isinstance(relocations, ecoscope.base.Relocations) assert "fixtime" in relocations assert "groupby_col" in relocations + check_time_is_parsed(relocations) def test_get_source_no_observations(er_io): @@ -54,6 +64,7 @@ def test_get_subjectsource_observations(er_io): assert isinstance(relocations, ecoscope.base.Relocations) assert "fixtime" in relocations assert "groupby_col" in relocations + check_time_is_parsed(relocations) # TODO Check this def test_get_subjectsource_no_observations(er_io): @@ -73,6 +84,7 @@ def test_get_subjectgroup_observations(er_io): def test_get_events(er_events_io): events = er_events_io.get_events(event_type=["e00ce1f6-f9f1-48af-93c9-fb89ec493b8a"]) assert not events.empty + check_time_is_parsed(events) def test_das_client_method(er_io): @@ -88,6 +100,7 @@ def test_get_patrols_datestr(er_io): patrols = er_io.get_patrols(since=since_str, until=until_str) assert len(patrols) > 0 + check_time_is_parsed(patrols) time_ranges = [ segment["time_range"] @@ -118,6 +131,7 @@ def test_get_patrols_with_type_value(er_io): if "patrol_type" in segment ] assert all(value == "ecoscope_patrol" for value in patrol_types) + check_time_is_parsed(patrols) def test_get_patrols_with_type_value_list(er_io): @@ -131,6 +145,7 @@ def test_get_patrols_with_type_value_list(er_io): if "patrol_type" in segment ] assert all(value in patrol_type_value_list for value in patrol_types) + check_time_is_parsed(patrols) def test_get_patrols_with_invalid_type_value(er_io): @@ -150,6 +165,7 @@ def test_get_patrol_events(er_io): assert "patrol_segment_id" in events assert "patrol_start_time" in events assert "time" in events + check_time_is_parsed(events) @patch("ecoscope.io.EarthRangerIO.get_patrols") @@ -276,6 +292,7 @@ def test_get_patrol_observations(er_io): include_subjectsource_details=False, ) assert not observations.empty + check_time_is_parsed(observations) def test_get_patrol_observations_with_patrol_details(er_io): @@ -295,6 +312,7 @@ def test_get_patrol_observations_with_patrol_details(er_io): assert "patrol_id" in observations.columns assert "patrol_title" in observations.columns pd.testing.assert_series_equal(observations["patrol_id"], observations["groupby_col"], check_names=False) + check_time_is_parsed(observations) def test_users(er_io):