diff --git a/.github/workflows/io_tests.yaml b/.github/workflows/io_tests.yaml new file mode 100644 index 00000000..02de3b77 --- /dev/null +++ b/.github/workflows/io_tests.yaml @@ -0,0 +1,15 @@ +name: Test IO + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "*" ] + +jobs: + dev: + uses: ./.github/workflows/tests.yml + with: + io: "io" + os: "ubuntu-latest" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e9aea145..63585078 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,4 +1,4 @@ -name: Tests +name: Tests Main on: push: @@ -7,42 +7,8 @@ on: branches: [ "*" ] jobs: - Test: - name: ${{ matrix.os }}, ${{ matrix.env }} - runs-on: ${{ matrix.os }} - defaults: - run: - shell: bash -leo pipefail {0} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - env: [environment.yml] - - steps: - - uses: actions/checkout@v4 - - - name: Setup Micromamba - uses: mamba-org/setup-micromamba@v2.0.2 - with: - environment-file: ${{ matrix.env }} - cache-environment: true - cache-downloads: true - init-shell: bash powershell - - - name: Install pip dependencies and our package - shell: bash -leo pipefail {0} - run: | - python -m pip install ".[all]" - - - name: Test - env: - ER_USERNAME: ${{ secrets.ER_USERNAME }} - ER_PASSWORD: ${{ secrets.ER_PASSWORD }} - EE_ACCOUNT: ${{ secrets.EE_ACCOUNT }} - EE_PRIVATE_KEY_DATA: ${{ secrets.EE_PRIVATE_KEY_DATA }} - run: | - pytest -v -r s --color=yes --cov=ecoscope --cov-append --cov-report=xml - - - name: Codecov - uses: codecov/codecov-action@v5 + dev: + uses: ./.github/workflows/tests.yml + with: + io: "not io" + os: "ubuntu-latest" \ No newline at end of file diff --git a/.github/workflows/test_all_daily.yaml b/.github/workflows/test_all_daily.yaml new file mode 100644 index 00000000..36d19cac --- /dev/null +++ b/.github/workflows/test_all_daily.yaml @@ -0,0 +1,36 @@ +name: Test Daily + +on: + schedule: + # Per https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#schedule + # Make this a weird time + - cron: "23 5 * * *" + workflow_dispatch: {} + + +jobs: + ubuntu: + uses: ./.github/workflows/tests.yml + with: + io: "io or not io" + os: "ubuntu-latest" + secrets: inherit + mac: + uses: ./.github/workflows/tests.yml + # The If Always/Needs combo here is so that these tests can be + # run sequentially but don't care about eachother failing + # We want to run sequentially to minimise the load on mep-dev + if: ${{ always() }} + needs: [ubuntu] + with: + io: "io or not io" + os: "macos-latest" + secrets: inherit + windows: + uses: ./.github/workflows/tests.yml + if: ${{ always() }} + needs: [ubuntu, mac] + with: + io: "io or not io" + os: "windows-latest" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..358dc0af --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,47 @@ +name: Tests + +on: + workflow_call: + inputs: + io: + type: string + required: true + os: + type: string + required: true + +jobs: + Test: + name: ${{ inputs.os }}, ${{ inputs.io }} + runs-on: ${{ inputs.os }} + defaults: + run: + shell: bash -leo pipefail {0} + strategy: + fail-fast: false + + steps: + - uses: actions/checkout@v4 + + - name: Setup Micromamba + uses: mamba-org/setup-micromamba@v2.0.2 + with: + environment-file: environment.yml + cache-environment: true + init-shell: bash powershell + + - name: Install our package + run: | + python -m pip install ".[all]" + + - name: Test + env: + ER_USERNAME: ${{ secrets.ER_USERNAME }} + ER_PASSWORD: ${{ secrets.ER_PASSWORD }} + EE_ACCOUNT: ${{ secrets.EE_ACCOUNT }} + EE_PRIVATE_KEY_DATA: ${{ secrets.EE_PRIVATE_KEY_DATA }} + run: | + pytest -v -r s -m "${{ inputs.io }}" --color=yes --cov=ecoscope --cov-append --cov-report=xml + + - name: Codecov + uses: codecov/codecov-action@v5 \ No newline at end of file diff --git a/ecoscope/io/async_earthranger.py b/ecoscope/io/async_earthranger.py index 4c02fc1e..25b39424 100644 --- a/ecoscope/io/async_earthranger.py +++ b/ecoscope/io/async_earthranger.py @@ -4,8 +4,7 @@ import asyncio import ecoscope -from ecoscope.io.utils import to_hex -from ecoscope.io.earthranger_utils import clean_kwargs, to_gdf, clean_time_cols +from ecoscope.io.earthranger_utils import clean_kwargs, to_gdf, clean_time_cols, to_hex from erclient.client import ERClientException, ERClientNotFound try: diff --git a/ecoscope/io/earthranger.py b/ecoscope/io/earthranger.py index fa30d8d3..a0966381 100644 --- a/ecoscope/io/earthranger.py +++ b/ecoscope/io/earthranger.py @@ -19,8 +19,9 @@ dataframe_to_dict, format_iso_time, to_gdf, + to_hex, + pack_columns, ) -from ecoscope.io.utils import pack_columns, to_hex class EarthRangerIO(ERClient): diff --git a/ecoscope/io/earthranger_utils.py b/ecoscope/io/earthranger_utils.py index a8fc7ea8..0d5c6924 100644 --- a/ecoscope/io/earthranger_utils.py +++ b/ecoscope/io/earthranger_utils.py @@ -1,3 +1,4 @@ +import typing import geopandas as gpd import pandas as pd from dateutil import parser @@ -50,3 +51,21 @@ def format_iso_time(date_string: str) -> str: return pd.to_datetime(date_string).isoformat() except ValueError: raise ValueError(f"Failed to parse timestamp'{date_string}'") + + +def to_hex(val, default="#ff0000"): + if val and not pd.isnull(val): + return "#{:02X}{:02X}{:02X}".format(*[int(i) for i in val.split(",")]) + return default + + +def pack_columns(dataframe: pd.DataFrame, columns: typing.List): + """This method would add all extra columns to single column""" + metadata_cols = list(set(dataframe.columns).difference(set(columns))) + + # To prevent additional column from being dropped, name the column metadata (rename it back). + if metadata_cols: + dataframe["metadata"] = dataframe[metadata_cols].to_dict(orient="records") + dataframe.drop(metadata_cols, inplace=True, axis=1) + dataframe.rename(columns={"metadata": "additional"}, inplace=True) + return dataframe diff --git a/ecoscope/io/utils.py b/ecoscope/io/utils.py index 8c5609bc..1a69fa28 100644 --- a/ecoscope/io/utils.py +++ b/ecoscope/io/utils.py @@ -1,34 +1,14 @@ import email import os import re -import typing import zipfile -import pandas as pd import requests from requests.adapters import HTTPAdapter from tqdm.auto import tqdm from urllib3.util import Retry -def to_hex(val, default="#ff0000"): - if val and not pd.isnull(val): - return "#{:02X}{:02X}{:02X}".format(*[int(i) for i in val.split(",")]) - return default - - -def pack_columns(dataframe: pd.DataFrame, columns: typing.List): - """This method would add all extra columns to single column""" - metadata_cols = list(set(dataframe.columns).difference(set(columns))) - - # To prevent additional column from being dropped, name the column metadata (rename it back). - if metadata_cols: - dataframe["metadata"] = dataframe[metadata_cols].to_dict(orient="records") - dataframe.drop(metadata_cols, inplace=True, axis=1) - dataframe.rename(columns={"metadata": "additional"}, inplace=True) - return dataframe - - def download_file(url, path, retries=2, overwrite_existing=False, chunk_size=1024, unzip=False, **request_kwargs): """ Download a file from a URL to a local path. If the path is a directory, the filename will be inferred from diff --git a/pyproject.toml b/pyproject.toml index 8dd567dd..c8154c3d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,6 +89,9 @@ filterwarnings = [ "ignore:distutils Version classes are deprecated. Use packaging.version instead.:DeprecationWarning", 'ignore:Feature.geometry\(\) is deprecated. Use Element.geometry\(\):DeprecationWarning', ] +markers = [ + "io", +] [tool.black] include = '\.pyi?$' diff --git a/tests/conftest.py b/tests/conftest.py index 0a80ab73..8628dbef 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,7 +5,6 @@ import geopandas as gpd import pandas as pd import pytest -from erclient.client import ERClientNotFound import ecoscope from ecoscope.base import Relocations @@ -18,31 +17,20 @@ def pytest_configure(config): os.makedirs("tests/outputs", exist_ok=True) - try: - EE_ACCOUNT = os.getenv("EE_ACCOUNT") - EE_PRIVATE_KEY_DATA = os.getenv("EE_PRIVATE_KEY_DATA") - if EE_ACCOUNT and EE_PRIVATE_KEY_DATA: - ee.Initialize(credentials=ee.ServiceAccountCredentials(EE_ACCOUNT, key_data=EE_PRIVATE_KEY_DATA)) - else: - ee.Initialize() - pytest.earthengine = True - except Exception: - pytest.earthengine = False - warnings.warn(Warning("Earth Engine can not be initialized. Skipping related tests...")) - - try: - pytest.earthranger = ecoscope.io.EarthRangerIO( - server=os.getenv("ER_SERVER", "https://mep-dev.pamdas.org"), - username=os.getenv("ER_USERNAME"), - password=os.getenv("ER_PASSWORD"), - ).login() - if not pytest.earthranger: - warnings.warn(Warning("EarthRanger_IO can not be initialized. Skipping related tests...")) - except ERClientNotFound: - pytest.earthranger = False - warnings.warn(Warning("EarthRanger_IO can not be initialized. Skipping related tests...")) + if config.inicfg.get("markers") == ["io"]: + try: + EE_ACCOUNT = os.getenv("EE_ACCOUNT") + EE_PRIVATE_KEY_DATA = os.getenv("EE_PRIVATE_KEY_DATA") + if EE_ACCOUNT and EE_PRIVATE_KEY_DATA: + ee.Initialize(credentials=ee.ServiceAccountCredentials(EE_ACCOUNT, key_data=EE_PRIVATE_KEY_DATA)) + else: + ee.Initialize() + pytest.earthengine = True + except Exception: + warnings.warn(Warning("Earth Engine can not be initialized.")) +@pytest.mark.io @pytest.fixture(scope="session") def er_io(): ER_SERVER = "https://mep-dev.pamdas.org" @@ -59,6 +47,7 @@ def er_io(): return er_io +@pytest.mark.io @pytest.fixture(scope="session") def er_events_io(): ER_SERVER = "https://mep-dev.pamdas.org" @@ -100,3 +89,11 @@ def aoi_gdf(): regions_gdf = gpd.GeoDataFrame.from_file(AOI_FILE).to_crs(4326) regions_gdf.set_index("ZONE", drop=True, inplace=True) return regions_gdf + + +@pytest.fixture +def sample_relocs(): + gdf = gpd.read_parquet("tests/sample_data/vector/sample_relocs.parquet") + gdf = ecoscope.io.earthranger_utils.clean_time_cols(gdf) + + return ecoscope.base.Relocations.from_gdf(gdf) diff --git a/tests/sample_data/vector/sample_relocs.parquet b/tests/sample_data/vector/sample_relocs.parquet index f043e54e..9769d8b0 100644 Binary files a/tests/sample_data/vector/sample_relocs.parquet and b/tests/sample_data/vector/sample_relocs.parquet differ diff --git a/tests/sample_data/vector/sample_single_relocs.parquet b/tests/sample_data/vector/sample_single_relocs.parquet new file mode 100644 index 00000000..f043e54e Binary files /dev/null and b/tests/sample_data/vector/sample_single_relocs.parquet differ diff --git a/tests/sample_data/vector/spatial_features.feather b/tests/sample_data/vector/spatial_features.feather new file mode 100644 index 00000000..59358d5a Binary files /dev/null and b/tests/sample_data/vector/spatial_features.feather differ diff --git a/tests/test_asyncearthranger_io.py b/tests/test_asyncearthranger_io.py index caa06937..47d6f6e4 100644 --- a/tests/test_asyncearthranger_io.py +++ b/tests/test_asyncearthranger_io.py @@ -6,11 +6,7 @@ import ecoscope from erclient import ERClientException -if not pytest.earthranger: - pytest.skip( - "Skipping tests because connection to EarthRanger is not available.", - allow_module_level=True, - ) +pytestmark = pytest.mark.io @pytest_asyncio.fixture diff --git a/tests/test_base.py b/tests/test_base.py index e39659a5..b6580e87 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -8,46 +8,44 @@ import ecoscope -@pytest.mark.skipif(not pytest.earthranger, reason="No connection to EarthRanger") -def test_trajectory_is_not_empty(er_io): +@pytest.fixture +def sample_single_relocs(): + gdf = gpd.read_parquet("tests/sample_data/vector/sample_single_relocs.parquet") + gdf = ecoscope.io.earthranger_utils.clean_time_cols(gdf) + + return ecoscope.base.Relocations.from_gdf(gdf) + + +def test_trajectory_is_not_empty(sample_relocs): # test there is actually data in trajectory - relocations = er_io.get_subjectgroup_observations(subject_group_name=er_io.GROUP_NAME) - trajectory = ecoscope.base.Trajectory.from_relocations(relocations) + trajectory = ecoscope.base.Trajectory.from_relocations(sample_relocs) assert not trajectory.empty -@pytest.mark.skipif(not pytest.earthranger, reason="No connection to EarthRanger") -def test_redundant_columns_in_trajectory(er_io): +def test_redundant_columns_in_trajectory(sample_relocs): # test there is no redundant column in trajectory - relocations = er_io.get_subjectgroup_observations(subject_group_name=er_io.GROUP_NAME) - trajectory = ecoscope.base.Trajectory.from_relocations(relocations) + trajectory = ecoscope.base.Trajectory.from_relocations(sample_relocs) assert "extra__fixtime" not in trajectory assert "extra___fixtime" not in trajectory assert "extra___geometry" not in trajectory -@pytest.mark.skipif(not pytest.earthranger, reason="No connection to EarthRanger") -def test_relocs_speedfilter(er_io): - relocations = er_io.get_subjectgroup_observations(subject_group_name=er_io.GROUP_NAME) +def test_relocs_speedfilter(sample_relocs): relocs_speed_filter = ecoscope.base.RelocsSpeedFilter(max_speed_kmhr=8) - relocs_after_filter = relocations.apply_reloc_filter(relocs_speed_filter) + relocs_after_filter = sample_relocs.apply_reloc_filter(relocs_speed_filter) relocs_after_filter.remove_filtered(inplace=True) - assert relocations.shape[0] != relocs_after_filter.shape[0] + assert sample_relocs.shape[0] != relocs_after_filter.shape[0] -@pytest.mark.skipif(not pytest.earthranger, reason="No connection to EarthRanger") -def test_relocs_distancefilter(er_io): - relocations = er_io.get_subjectgroup_observations(subject_group_name=er_io.GROUP_NAME) +def test_relocs_distancefilter(sample_relocs): relocs_speed_filter = ecoscope.base.RelocsDistFilter(min_dist_km=1.0, max_dist_km=6.0) - relocs_after_filter = relocations.apply_reloc_filter(relocs_speed_filter) + relocs_after_filter = sample_relocs.apply_reloc_filter(relocs_speed_filter) relocs_after_filter.remove_filtered(inplace=True) - assert relocations.shape[0] != relocs_after_filter.shape[0] + assert sample_relocs.shape[0] != relocs_after_filter.shape[0] -@pytest.mark.skipif(not pytest.earthranger, reason="No connection to EarthRanger") -def test_relocations_from_gdf_preserve_fields(er_io): - relocations = er_io.get_subjectgroup_observations(subject_group_name=er_io.GROUP_NAME) - gpd.testing.assert_geodataframe_equal(relocations, ecoscope.base.Relocations.from_gdf(relocations)) +def test_relocations_from_gdf_preserve_fields(sample_relocs): + gpd.testing.assert_geodataframe_equal(sample_relocs, ecoscope.base.Relocations.from_gdf(sample_relocs)) def test_trajectory_properties(movebank_relocations): @@ -235,24 +233,16 @@ def test_apply_traj_filter(movebank_relocations): assert filtered["speed_kmhr"].max() <= max_speed -@pytest.fixture -def sample_relocs(): - gdf = gpd.read_parquet("tests/sample_data/vector/sample_relocs.parquet") - gdf = ecoscope.io.earthranger_utils.clean_time_cols(gdf) - - return ecoscope.base.Relocations.from_gdf(gdf) - - -def test_trajectory_with_single_relocation(sample_relocs): - assert len(sample_relocs["extra__subject_id"].unique()) == 3 - trajectory = ecoscope.base.Trajectory.from_relocations(sample_relocs) +def test_trajectory_with_single_relocation(sample_single_relocs): + assert len(sample_single_relocs["extra__subject_id"].unique()) == 3 + trajectory = ecoscope.base.Trajectory.from_relocations(sample_single_relocs) assert not trajectory.empty assert len(trajectory["extra__subject_id"].unique()) == 2 -def test_trajectory_preserves_column_dtypes(sample_relocs): - before = sample_relocs.dtypes - trajectory = ecoscope.base.Trajectory.from_relocations(sample_relocs) +def test_trajectory_preserves_column_dtypes(sample_single_relocs): + before = sample_single_relocs.dtypes + trajectory = ecoscope.base.Trajectory.from_relocations(sample_single_relocs) after = trajectory.dtypes for col in before.index: diff --git a/tests/test_earthranger_io.py b/tests/test_earthranger_io.py index e04d0acb..9875c191 100644 --- a/tests/test_earthranger_io.py +++ b/tests/test_earthranger_io.py @@ -11,11 +11,7 @@ import ecoscope from erclient import ERClientException -if not pytest.earthranger: - pytest.skip( - "Skipping tests because connection to EarthRanger is not available.", - allow_module_level=True, - ) +pytestmark = pytest.mark.io def test_get_subject_observations(er_io): diff --git a/tests/test_ecomap.py b/tests/test_ecomap.py index c4955b3c..e6f4d553 100644 --- a/tests/test_ecomap.py +++ b/tests/test_ecomap.py @@ -144,7 +144,7 @@ def test_add_save_image(): assert isinstance(m.deck_widgets[3], SaveImageWidget) -@pytest.mark.skipif(not pytest.earthengine, reason="No connection to EarthEngine.") +@pytest.mark.io def test_add_ee_layer_image(): m = EcoMap() vis_params = {"min": 0, "max": 4000, "opacity": 0.5, "palette": ["006633", "E5FFCC", "662A00", "D8D8D8", "F5F5F5"]} @@ -154,7 +154,7 @@ def test_add_ee_layer_image(): assert isinstance(m.layers[0], BitmapTileLayer) -@pytest.mark.skipif(not pytest.earthengine, reason="No connection to EarthEngine.") +@pytest.mark.io def test_add_ee_layer_image_collection(): m = EcoMap() vis_params = {"min": 0, "max": 4000, "opacity": 0.5} @@ -165,7 +165,7 @@ def test_add_ee_layer_image_collection(): assert m.layers[0].tile_size == 256 -@pytest.mark.skipif(not pytest.earthengine, reason="No connection to EarthEngine.") +@pytest.mark.io def test_add_ee_layer_feature_collection(): m = EcoMap() vis_params = {"min": 0, "max": 4000, "opacity": 0.5, "palette": ["006633", "E5FFCC", "662A00", "D8D8D8", "F5F5F5"]} @@ -175,7 +175,7 @@ def test_add_ee_layer_feature_collection(): assert isinstance(m.layers[0], BitmapTileLayer) -@pytest.mark.skipif(not pytest.earthengine, reason="No connection to EarthEngine.") +@pytest.mark.io def test_add_ee_layer_geometry(): m = EcoMap() rectangle = ee.Geometry.Rectangle([-40, -20, 40, 20]) diff --git a/tests/test_eetools.py b/tests/test_eetools.py index 5b42f06e..8f710c4f 100644 --- a/tests/test_eetools.py +++ b/tests/test_eetools.py @@ -4,8 +4,7 @@ import ecoscope -if not pytest.earthengine: - pytest.skip("Skipping tests because connection to Earth Engine is not available.", allow_module_level=True) +pytestmark = pytest.mark.io def test_albedo_anomaly(aoi_gdf): diff --git a/tests/test_io_utils.py b/tests/test_io_utils.py index 8cd739b8..80f09b56 100644 --- a/tests/test_io_utils.py +++ b/tests/test_io_utils.py @@ -10,6 +10,8 @@ import ecoscope +pytestmark = pytest.mark.io + def test_download_file_github_csv(): ECOSCOPE_RAW = "https://raw.githubusercontent.com/wildlife-dynamics/ecoscope/master" diff --git a/tests/test_proiximity.py b/tests/test_proximity.py similarity index 54% rename from tests/test_proiximity.py rename to tests/test_proximity.py index a68404b5..bbd0e293 100644 --- a/tests/test_proiximity.py +++ b/tests/test_proximity.py @@ -1,19 +1,21 @@ import pytest +import geopandas as gpd from ecoscope.analysis.proximity import SpatialFeature, Proximity, ProximityProfile from ecoscope.base import Trajectory -@pytest.mark.skipif(not pytest.earthranger, reason="No connection to EarthRanger") -def test_proximity(er_io): - er_features = er_io.get_spatial_features_group(spatial_features_group_id="15698426-7e0f-41df-9bc3-495d87e2e097") +@pytest.fixture +def sample_spatial_features(): + return gpd.read_feather("tests/sample_data/vector/spatial_features.feather") + +def test_proximity(sample_relocs, sample_spatial_features): prox_profile = ProximityProfile([]) - for row in er_features.iterrows(): + for row in sample_spatial_features.iterrows(): prox_profile.spatial_features.append(SpatialFeature(row[1]["name"], row[1]["pk"], row[1]["geometry"])) - relocations = er_io.get_subjectgroup_observations(subject_group_name=er_io.GROUP_NAME) - trajectory = Trajectory.from_relocations(relocations) + trajectory = Trajectory.from_relocations(sample_relocs) proximity_events = Proximity.calculate_proximity(proximity_profile=prox_profile, trajectory=trajectory) diff --git a/tests/test_seasons.py b/tests/test_seasons.py index 76915a02..404e78cb 100644 --- a/tests/test_seasons.py +++ b/tests/test_seasons.py @@ -4,14 +4,8 @@ from ecoscope import plotting from ecoscope.analysis import seasons -if not pytest.earthengine: - pytest.skip( - "Skipping tests because connection to Earth Engine is not available.", - allow_module_level=True, - ) - -@pytest.mark.skipif(not pytest.earthengine, reason="No connection to EarthEngine.") +@pytest.mark.io def test_seasons(): gdf = gpd.read_file("tests/sample_data/vector/AOI_sites.gpkg").to_crs(4326)