Fix scanner (#18)

* test: Get recent data

Verify InRange searching for recent data. So many unrelated things can
go wrong to fail this test, but should work just fine most of the time.

* fix: inrange() was renamed to matchup()

* fix: bloom_filter for multiple sensors

It was missing dL_tol on recursive calls.

* Bump version: 0.0.9 → 0.0.10

* fix: typo

* fix: Inconsistent title between HISTORY & README

CITATION.cff

@@ -6,7 +6,7 @@ authors:
     family-names: Castelao
     affiliation: Scripps Institution of Oceanography - UC San Diego
     orcid: https://orcid.org/0000-0002-6765-0708
-version: 0.0.9
+version: 0.0.10
 doi: 10.5281/zenodo.4646628
 date-released: 2021-03-21
 repository-code: https://github.com/castelao/CoTeDe

HISTORY.rst

@@ -4,16 +4,19 @@ History
 
 Before release 0.1 expect gaps in the history description
 
+-----
 0.0.9
 -----
 
 * Stable concurrent searches using loky
 
+------------------
 0.0.1 (2020-02-24)
 ------------------
 
 * First release on PyPI.
 
+------------------------
 pre-release (since 2018)
 ------------------------
 

OceanColor/__init__.py

@@ -7,7 +7,7 @@
 
 __author__ = """Guilherme Castelão"""
 __email__ = "[email protected]"
-__version__ = "0.0.9"
+__version__ = "0.0.10"
 
 # Recent OSX requires this environment variable to run parallel processes
 if sys.platform == "darwin":

OceanColor/cmr.py

@@ -209,7 +209,7 @@ def bloom_filter(
     """
     if isinstance(sensor, list):
         for s in sensor:
-            filenames = bloom_filter(track, s, dtype, dt_tol)
+            filenames = bloom_filter(track, s, dtype, dt_tol, dL_tol)
             yield from filenames
         return
 

OceanColor/inrange.py

@@ -145,7 +145,7 @@ def scanner_threading(self, queue, parent, npes, track, sensor, dtype, dt_tol, d
                 return
             results.append(
                 threading.Thread(
-                    target=inrange, args=(track, ds, dL_tol, dt_tol, queue)
+                    target=matchup, args=(track, ds, dL_tol, dt_tol, queue)
                 )
             )
             results[-1].start()
@@ -185,7 +185,7 @@ def scanner(self, queue, parent, npes, track, sensor, dtype, dt_tol, dL_tol):
                 if not parent.is_alive():
                     return
                 module_logger.debug("Submitting a new inrange process")
-                results.append(executor.submit(inrange, track, ds, dL_tol, dt_tol))
+                results.append(executor.submit(matchup, track, ds, dL_tol, dt_tol))
 
             for tmp in (r.result(timeout) for r in results):
                 if not parent.is_alive():

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.0.9
+current_version = 0.0.10
 commit = True
 tag = True
 
@@ -29,3 +29,4 @@ test = pytest
 
 [tool:pytest]
 collect_ignore = ['setup.py']
+

setup.py

@@ -50,7 +50,7 @@ def requirements():
     test_suite='tests',
     tests_require=test_requirements,
     url='https://github.com/castelao/OceanColor',
-    version="0.0.9",
+    version="0.0.10",
     zip_safe=False,
     extras_require = {
         'parallel': ["loky>=2.8"],

tests/test_cmr.py

@@ -78,6 +78,18 @@ def test_bloom_filter_spaced_target():
     assert len(results) < 5
     assert len(results) == 3
 
+
+def test_bloom_multiple_sensors():
+    track = [{"time": datetime64("2019-05-01"), "lat": 18, "lon": 38}]
+    filter = bloom_filter(
+        pd.DataFrame(track),
+        sensor=["aqua", "terra", "snpp"],
+        dtype="L2",
+        dt_tol=timedelta64(36, "h"),
+        dL_tol=10e3
+    )
+    assert len([f for f in filter]) > 0
+
 def test_search_criteria():
     search = search_criteria(sensor="aqua", dtype="L2")
     assert search["short_name"] == "MODISA_L2_OC"

tests/test_inrange.py

@@ -1,8 +1,13 @@
 """Test module inrange
 """
 
+from datetime import datetime
+import tempfile
+
 from numpy import datetime64, timedelta64
 import os
+
+import pandas as pd
 from pandas import DataFrame
 
 from OceanColor.inrange import matchup_L2, matchup_L3m, matchup
@@ -80,6 +85,34 @@ def test_matchup():
     assert data.size == 42
 
 
+def test_InRange_recent():
+    """Find recent in range
+
+    By using FileSystem in a temporary directory guarantees that the cache
+    is not been used. At some point a typo in the InRange's scanner was
+    missed by the tests since it was acessing the cache without actually
+    downloading it.
+    """
+    track = DataFrame(
+        [
+            {
+                "time": datetime64(datetime.utcnow()) - timedelta64(15, "D"),
+                "lat": 35.6,
+                "lon": -126.81,
+            }
+        ]
+    )
+
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        matchup = InRange(username, password, tmpdirname, npes=3)
+        matchup.search(
+            track, sensor="snpp", dtype="L2", dt_tol=timedelta64(12, "h"), dL_tol=10e3
+        )
+        output = pd.concat([m for m in matchup])
+
+    assert len(output) > 0
+
+
 def test_InRange_early_termination():
   """Terminate before consuming or even finished searching
   """