Test improvements (#341)

.github/workflows/io_tests.yaml

@@ -0,0 +1,15 @@
+name: Test IO
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "*" ]
+
+jobs:
+  dev:
+    uses: ./.github/workflows/tests.yml
+    with:
+      io: "io"
+      os: "ubuntu-latest"
+    secrets: inherit

.github/workflows/main.yml

@@ -1,4 +1,4 @@
-name: Tests
+name: Tests Main
 
 on:
   push:
@@ -7,42 +7,8 @@ on:
     branches: [ "*" ]
 
 jobs:
-  Test:
-    name: ${{ matrix.os }}, ${{ matrix.env }}
-    runs-on: ${{ matrix.os }}
-    defaults:
-      run:
-        shell: bash -leo pipefail {0}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
-        env: [environment.yml]
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Setup Micromamba
-        uses: mamba-org/[email protected]
-        with:
-          environment-file: ${{ matrix.env }}
-          cache-environment: true
-          cache-downloads: true
-          init-shell: bash powershell
-
-      - name: Install pip dependencies and our package
-        shell: bash -leo pipefail {0}
-        run: |
-          python -m pip install ".[all]"
-
-      - name: Test
-        env:
-          ER_USERNAME: ${{ secrets.ER_USERNAME }}
-          ER_PASSWORD: ${{ secrets.ER_PASSWORD }}
-          EE_ACCOUNT: ${{ secrets.EE_ACCOUNT }}
-          EE_PRIVATE_KEY_DATA: ${{ secrets.EE_PRIVATE_KEY_DATA }}
-        run: |
-          pytest -v -r s --color=yes --cov=ecoscope --cov-append --cov-report=xml
-      
-      - name: Codecov
-        uses: codecov/codecov-action@v5
+  dev:
+    uses: ./.github/workflows/tests.yml
+    with:
+      io: "not io"
+      os: "ubuntu-latest"

.github/workflows/test_all_daily.yaml

@@ -0,0 +1,36 @@
+name: Test Daily
+
+on:
+  schedule:
+    # Per https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#schedule
+    # Make this a weird time
+    - cron: "23 5 * * *"
+  workflow_dispatch: {}
+
+
+jobs:
+  ubuntu:
+    uses: ./.github/workflows/tests.yml
+    with:
+      io: "io or not io"
+      os: "ubuntu-latest"
+    secrets: inherit
+  mac:
+    uses: ./.github/workflows/tests.yml
+    # The If Always/Needs combo here is so that these tests can be 
+    # run sequentially but don't care about eachother failing
+    # We want to run sequentially to minimise the load on mep-dev
+    if: ${{ always() }}
+    needs: [ubuntu]
+    with:
+      io: "io or not io"
+      os: "macos-latest"
+    secrets: inherit
+  windows:
+    uses: ./.github/workflows/tests.yml
+    if: ${{ always() }}
+    needs: [ubuntu, mac]
+    with:
+      io: "io or not io"
+      os: "windows-latest"
+    secrets: inherit

.github/workflows/tests.yml

@@ -0,0 +1,47 @@
+name: Tests
+
+on:
+  workflow_call:
+    inputs:
+      io:
+        type: string
+        required: true
+      os:
+        type: string
+        required: true
+
+jobs:
+  Test:
+    name: ${{ inputs.os }}, ${{ inputs.io }}
+    runs-on: ${{ inputs.os }}
+    defaults:
+      run:
+        shell: bash -leo pipefail {0}
+    strategy:
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Micromamba
+        uses: mamba-org/[email protected]
+        with:
+          environment-file: environment.yml
+          cache-environment: true
+          init-shell: bash powershell
+
+      - name: Install our package
+        run: |
+          python -m pip install ".[all]"
+
+      - name: Test
+        env:
+          ER_USERNAME: ${{ secrets.ER_USERNAME }}
+          ER_PASSWORD: ${{ secrets.ER_PASSWORD }}
+          EE_ACCOUNT: ${{ secrets.EE_ACCOUNT }}
+          EE_PRIVATE_KEY_DATA: ${{ secrets.EE_PRIVATE_KEY_DATA }}
+        run: |
+          pytest -v -r s -m "${{ inputs.io }}" --color=yes --cov=ecoscope --cov-append --cov-report=xml
+      
+      - name: Codecov
+        uses: codecov/codecov-action@v5

ecoscope/io/async_earthranger.py

@@ -4,8 +4,7 @@
 import asyncio
 
 import ecoscope
-from ecoscope.io.utils import to_hex
-from ecoscope.io.earthranger_utils import clean_kwargs, to_gdf, clean_time_cols
+from ecoscope.io.earthranger_utils import clean_kwargs, to_gdf, clean_time_cols, to_hex
 from erclient.client import ERClientException, ERClientNotFound
 
 try:

ecoscope/io/earthranger.py

@@ -19,8 +19,9 @@
     dataframe_to_dict,
     format_iso_time,
     to_gdf,
+    to_hex,
+    pack_columns,
 )
-from ecoscope.io.utils import pack_columns, to_hex
 
 
 class EarthRangerIO(ERClient):

ecoscope/io/earthranger_utils.py

@@ -1,3 +1,4 @@
+import typing
 import geopandas as gpd
 import pandas as pd
 from dateutil import parser
@@ -50,3 +51,21 @@ def format_iso_time(date_string: str) -> str:
         return pd.to_datetime(date_string).isoformat()
     except ValueError:
         raise ValueError(f"Failed to parse timestamp'{date_string}'")
+
+
+def to_hex(val, default="#ff0000"):
+    if val and not pd.isnull(val):
+        return "#{:02X}{:02X}{:02X}".format(*[int(i) for i in val.split(",")])
+    return default
+
+
+def pack_columns(dataframe: pd.DataFrame, columns: typing.List):
+    """This method would add all extra columns to single column"""
+    metadata_cols = list(set(dataframe.columns).difference(set(columns)))
+
+    # To prevent additional column from being dropped, name the column metadata (rename it back).
+    if metadata_cols:
+        dataframe["metadata"] = dataframe[metadata_cols].to_dict(orient="records")
+        dataframe.drop(metadata_cols, inplace=True, axis=1)
+        dataframe.rename(columns={"metadata": "additional"}, inplace=True)
+    return dataframe

ecoscope/io/utils.py

@@ -1,34 +1,14 @@
 import email
 import os
 import re
-import typing
 import zipfile
 
-import pandas as pd
 import requests
 from requests.adapters import HTTPAdapter
 from tqdm.auto import tqdm
 from urllib3.util import Retry
 
 
-def to_hex(val, default="#ff0000"):
-    if val and not pd.isnull(val):
-        return "#{:02X}{:02X}{:02X}".format(*[int(i) for i in val.split(",")])
-    return default
-
-
-def pack_columns(dataframe: pd.DataFrame, columns: typing.List):
-    """This method would add all extra columns to single column"""
-    metadata_cols = list(set(dataframe.columns).difference(set(columns)))
-
-    # To prevent additional column from being dropped, name the column metadata (rename it back).
-    if metadata_cols:
-        dataframe["metadata"] = dataframe[metadata_cols].to_dict(orient="records")
-        dataframe.drop(metadata_cols, inplace=True, axis=1)
-        dataframe.rename(columns={"metadata": "additional"}, inplace=True)
-    return dataframe
-
-
 def download_file(url, path, retries=2, overwrite_existing=False, chunk_size=1024, unzip=False, **request_kwargs):
     """
     Download a file from a URL to a local path. If the path is a directory, the filename will be inferred from

pyproject.toml

@@ -89,6 +89,9 @@ filterwarnings = [
     "ignore:distutils Version classes are deprecated. Use packaging.version instead.:DeprecationWarning",
     'ignore:Feature.geometry\(\) is deprecated. Use Element.geometry\(\):DeprecationWarning',
 ]
+markers = [
+    "io",
+]
 
 [tool.black]
 include = '\.pyi?$'

tests/conftest.py

@@ -5,7 +5,6 @@
 import geopandas as gpd
 import pandas as pd
 import pytest
-from erclient.client import ERClientNotFound
 
 import ecoscope
 from ecoscope.base import Relocations
@@ -18,31 +17,20 @@ def pytest_configure(config):
 
     os.makedirs("tests/outputs", exist_ok=True)
 
-    try:
-        EE_ACCOUNT = os.getenv("EE_ACCOUNT")
-        EE_PRIVATE_KEY_DATA = os.getenv("EE_PRIVATE_KEY_DATA")
-        if EE_ACCOUNT and EE_PRIVATE_KEY_DATA:
-            ee.Initialize(credentials=ee.ServiceAccountCredentials(EE_ACCOUNT, key_data=EE_PRIVATE_KEY_DATA))
-        else:
-            ee.Initialize()
-        pytest.earthengine = True
-    except Exception:
-        pytest.earthengine = False
-        warnings.warn(Warning("Earth Engine can not be initialized. Skipping related tests..."))
-
-    try:
-        pytest.earthranger = ecoscope.io.EarthRangerIO(
-            server=os.getenv("ER_SERVER", "https://mep-dev.pamdas.org"),
-            username=os.getenv("ER_USERNAME"),
-            password=os.getenv("ER_PASSWORD"),
-        ).login()
-        if not pytest.earthranger:
-            warnings.warn(Warning("EarthRanger_IO can not be initialized. Skipping related tests..."))
-    except ERClientNotFound:
-        pytest.earthranger = False
-        warnings.warn(Warning("EarthRanger_IO can not be initialized. Skipping related tests..."))
+    if config.inicfg.get("markers") == ["io"]:
+        try:
+            EE_ACCOUNT = os.getenv("EE_ACCOUNT")
+            EE_PRIVATE_KEY_DATA = os.getenv("EE_PRIVATE_KEY_DATA")
+            if EE_ACCOUNT and EE_PRIVATE_KEY_DATA:
+                ee.Initialize(credentials=ee.ServiceAccountCredentials(EE_ACCOUNT, key_data=EE_PRIVATE_KEY_DATA))
+            else:
+                ee.Initialize()
+            pytest.earthengine = True
+        except Exception:
+            warnings.warn(Warning("Earth Engine can not be initialized."))
 
 
+@pytest.mark.io
 @pytest.fixture(scope="session")
 def er_io():
     ER_SERVER = "https://mep-dev.pamdas.org"
@@ -59,6 +47,7 @@ def er_io():
     return er_io
 
 
+@pytest.mark.io
 @pytest.fixture(scope="session")
 def er_events_io():
     ER_SERVER = "https://mep-dev.pamdas.org"
@@ -100,3 +89,11 @@ def aoi_gdf():
     regions_gdf = gpd.GeoDataFrame.from_file(AOI_FILE).to_crs(4326)
     regions_gdf.set_index("ZONE", drop=True, inplace=True)
     return regions_gdf
+
+
+@pytest.fixture
+def sample_relocs():
+    gdf = gpd.read_parquet("tests/sample_data/vector/sample_relocs.parquet")
+    gdf = ecoscope.io.earthranger_utils.clean_time_cols(gdf)
+
+    return ecoscope.base.Relocations.from_gdf(gdf)

tests/test_asyncearthranger_io.py

@@ -6,11 +6,7 @@
 import ecoscope
 from erclient import ERClientException
 
-if not pytest.earthranger:
-    pytest.skip(
-        "Skipping tests because connection to EarthRanger is not available.",
-        allow_module_level=True,
-    )
+pytestmark = pytest.mark.io
 
 
 @pytest_asyncio.fixture