Merge pull request #140 from developmentseed/nullfix

add fix for using is null
developmentseed · Nov 15, 2023 · f5d4c57 · f5d4c57 · github-actions · Nov 15, 2023
2 parents e921266 + 90db2a1
commit f5d4c57
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 4 deletions.
diff --git a/tests/fixtures/my_data.sql b/tests/fixtures/my_data.sql
@@ -14,7 +14,7 @@ INSERT INTO "public"."my_data" ("geom" , "id", "datetime", "decimal", "numeric")
 INSERT INTO "public"."my_data" ("geom" , "id", "datetime", "decimal", "numeric") VALUES ('0103000020E61000000100000013000000C0155236C40A38C052F1FFE1D8C75340B244B5A16EC837C014EBB5CD0CC4534073D712F2414F37C0D3BCE3141DBD5340FE41CA2BA27737C016B27D9C8ABB5340A2728C64C30A38C03BFB4402D0B553400C6AB4D7723A3DC0BDA377861D82534058CA32C4B15E3DC062105839B48053402A2097D1F19641C0EAE96F4E58CC5340F0A7C64B379941C07F6ABC7493CC5340E11AE2531A8741C01F2670501DCE5340CED31A45F57241C03EC92059D3CF534009E08D47F1E83FC0EAC3384350F05340DFE755925F713EC036A2858243005440ACEF9DFAC14B3DC0E950B3BEBB0C544034C8A112A4243DC064CC7707650E5440F602E719D4063DC0AE877727A90F54400A68226C78FA3CC0234A7B832F105440A630DBCBFBF43CC0E22ABE1BDF0F5440C0155236C40A38C052F1FFE1D8C75340', '2', '2004-10-21 10:23:54', -5.68, -5.68);
 INSERT INTO "public"."my_data" ("geom" , "id", "datetime", "decimal", "numeric") VALUES ('0103000020E610000001000000110000001B2CBE53855542C051F99E0E805D534049A5CD2EAE0644C03857A7D846865340462575029A0844C0A60A46257586534063B4EEABC4F943C08D992E511D8853409C72BC6BC5E843C0920AAB5C038A5340721D3749863342C03D0220C7DABA53402A2097D1F19641C0EAE96F4E58CC5340E11AE2531A8741C01F2670501DCE534068226C787A7541C0075F984C15D05340CED31A45F57241C03EC92059D3CF534048E17A14AE173DC06B2BF697DD8353400C6AB4D7723A3DC0BDA377861D825340A03E0335AD283FC0314A54553C6953409C6F1F2DEA1541C00EA6095E6A425340BEC11726532541C0BE9F1A2FDD405340EB51B81E853342C0302C67AA4C5A53401B2CBE53855542C051F99E0E805D5340', '3', '2004-10-22 10:23:54', 98, 98);
 INSERT INTO "public"."my_data" ("geom" , "id", "datetime", "decimal", "numeric") VALUES ('0103000020E610000001000000110000000A4C8422590E46C0B656FB86F03B5340D5E76A2BF60F46C0075F984C153C5340FA28B2217F0346C0CE0A257ADB3D5340BEE6287052F545C01AA33BF2DF3F5340F25A937BB7D244C009CB92853C69534049A5CD2EAE0644C03857A7D84686534063B4EEABC4F943C08D992E511D88534034A2B437F8EA43C0F54A5986388A53409C72BC6BC5E843C0920AAB5C038A534050AF9465883342C0363B85F6B5605340D43E0032881142C02A5884BF7F5D5340F4FDD478E90641C007F01648504453409C6F1F2DEA1541C00EA6095E6A4253404E4E9C88873342C06DC6E4C7471E53403EDF52396E3443C0DC9EAF2DC7FD524044696FF0854143C032772D211FFC52400A4C8422590E46C0B656FB86F03B5340', '4', '2004-10-23 10:23:54', 7.55526, 7.55526);
-INSERT INTO "public"."my_data" ("geom" , "id", "datetime", "decimal", "numeric") VALUES ('0103000020E6100000010000000D000000BBE9944235C347C0EBF06E7961EE52406ADE718A8EC447C0D122DBF97EEE5240942D6301ECB947C05B59871F60F0524086CAEEF61AAE47C0BDEF3BBB76F252400A4C8422590E46C0B656FB86F03B5340FA28B2217F0346C0CE0A257ADB3D534057EC2FBB27F745C02B1895D409405340BEE6287052F545C01AA33BF2DF3F53401D386744692743C07958A835CDFF52403EDF52396E3443C0DC9EAF2DC7FD5240B9E39237FD0645C0574B4E2543B552400AD7A3703D1245C03A234A7B83B35240BBE9944235C347C0EBF06E7961EE5240', '5', '2004-10-24 10:23:54', -78.56, -78.56);
+INSERT INTO "public"."my_data" ("geom" , "id", "datetime", "decimal", "numeric") VALUES ('0103000020E6100000010000000D000000BBE9944235C347C0EBF06E7961EE52406ADE718A8EC447C0D122DBF97EEE5240942D6301ECB947C05B59871F60F0524086CAEEF61AAE47C0BDEF3BBB76F252400A4C8422590E46C0B656FB86F03B5340FA28B2217F0346C0CE0A257ADB3D534057EC2FBB27F745C02B1895D409405340BEE6287052F545C01AA33BF2DF3F53401D386744692743C07958A835CDFF52403EDF52396E3443C0DC9EAF2DC7FD5240B9E39237FD0645C0574B4E2543B552400AD7A3703D1245C03A234A7B83B35240BBE9944235C347C0EBF06E7961EE5240', '5', '2004-10-24 10:23:54', -78.56, null);
 ALTER TABLE public.my_data ADD COLUMN otherdt timestamptz;
 ALTER TABLE public.my_data ADD COLUMN othergeom geometry;
 UPDATE my_data SET otherdt=datetime+'1 year'::interval, othergeom=st_pointonsurface(geom);

diff --git a/tests/routes/test_items.py b/tests/routes/test_items.py
@@ -327,6 +327,31 @@ def test_items_properties_filter_cql2(app):
     assert body["features"][0]["properties"]["row"] == 10
     Items.model_validate(body)
 
+    filter_query = {"op": "isNull", "args": [{"property": "numeric"}]}
+    response = app.get(
+        f"/collections/public.my_data/items?filter-lang=cql2-json&filter=&filter={json.dumps(filter_query)}"
+    )
+    assert response.status_code == 200
+    assert response.headers["content-type"] == "application/geo+json"
+    body = response.json()
+    assert len(body["features"]) == 1
+    assert body["numberMatched"] == 1
+    assert body["numberReturned"] == 1
+    assert body["features"][0]["properties"]["id"] == "5"
+    Items.model_validate(body)
+
+    response = app.get(
+        "/collections/public.my_data/items?filter-lang=cql2-text&filter=&filter=numeric IS NULL"
+    )
+    assert response.status_code == 200
+    assert response.headers["content-type"] == "application/geo+json"
+    body = response.json()
+    assert len(body["features"]) == 1
+    assert body["numberMatched"] == 1
+    assert body["numberReturned"] == 1
+    assert body["features"][0]["properties"]["id"] == "5"
+    Items.model_validate(body)
+
 
 def test_items_geo_filter_cql2(app):
     """Test CQL2 geo filter."""

diff --git a/tipg/filter/evaluate.py b/tipg/filter/evaluate.py
@@ -54,7 +54,9 @@ def in_(self, node, lhs, *options):  # noqa: D102
 
     @handle(ast.IsNull)
     def null(self, node, lhs):  # noqa: D102
-        return filters.runop(lhs, None, "is_null", node.not_)
+        if isinstance(lhs, list):
+            lhs = filters.attribute(lhs[0].name, self.field_mapping)
+        return filters.isnull(lhs)
 
     # @handle(ast.ExistsPredicateNode)
     # def exists(self, node, lhs):

diff --git a/tipg/filter/filters.py b/tipg/filter/filters.py
@@ -35,8 +35,6 @@ class Operator:
     """Filter Operators."""
 
     OPERATORS: Dict[str, Callable] = {
-        "is_null": lambda f, a=None: f.is_(None),
-        "is_not_null": lambda f, a=None: f.isnot(None),
         "==": lambda f, a: f == a,
         "=": lambda f, a: f == a,
         "eq": lambda f, a: f == a,
@@ -317,6 +315,11 @@ def attribute(name: str, fields: List[str]):
         raise TypeError(f"Field {name} not in table.")
 
 
+def isnull(lhs):
+    """null value."""
+    return lhs.is_(V("NULL"))
+
+
 def literal(value):
     """literal value."""
     return value
Benchmark suite	Current: `f5d4c57`	Previous: `e921266`	Ratio
`tests/benchmarks.py::test_benchmark_queryables`	`298.2872240037227` iter/sec (`stddev: 0.0013712089721707593`)	`439.300669384239` iter/sec (`stddev: 0.00011051197326519732`)	`1.47`
`tests/benchmarks.py::test_benchmark_items[geojson-1]`	`103.12735570809903` iter/sec (`stddev: 0.002123457622727346`)	`134.19494161802075` iter/sec (`stddev: 0.00031980952054504283`)	`1.30`
`tests/benchmarks.py::test_benchmark_items[csv-10]`	`75.1566828193599` iter/sec (`stddev: 0.001912450490465613`)	`97.98819309229839` iter/sec (`stddev: 0.0011802128905443144`)	`1.30`
`tests/benchmarks.py::test_benchmark_items[csv-200]`	`14.402675685305299` iter/sec (`stddev: 0.0033780917877960026`)	`19.292190253325632` iter/sec (`stddev: 0.0024498010819929248`)	`1.34`
`tests/benchmarks.py::test_benchmark_items[csv-250]`	`11.640403522574845` iter/sec (`stddev: 0.00479146804929302`)	`15.691931612233255` iter/sec (`stddev: 0.007668066831190365`)	`1.35`
`tests/benchmarks.py::test_benchmark_tile[4/8/5-WGS1984Quad]`	`86.41023099894556` iter/sec (`stddev: 0.0032103252880456234`)	`145.67015910157846` iter/sec (`stddev: 0.0006185525821461466`)	`1.69`
`tests/benchmarks.py::test_benchmark_tile[4/8/5-WebMercatorQuad]`	`61.93057139236294` iter/sec (`stddev: 0.015458616069852672`)	`108.35328327960536` iter/sec (`stddev: 0.0006943120639907678`)	`1.75`
`tests/benchmarks.py::test_benchmark_tile[6/33/25-WGS1984Quad]`	`102.60389826951581` iter/sec (`stddev: 0.0031372282494998255`)	`195.99017972973544` iter/sec (`stddev: 0.0005625669112904051`)	`1.91`
`tests/benchmarks.py::test_benchmark_tile[6/33/25-WebMercatorQuad]`	`110.13926008037951` iter/sec (`stddev: 0.002769751510082749`)	`187.1695772009401` iter/sec (`stddev: 0.00040874841401580017`)	`1.70`
`tests/benchmarks.py::test_benchmark_tilematrixset_endpoints[/tileMatrixSets]`	`284.879652097171` iter/sec (`stddev: 0.0015284162720615405`)	`441.1386174977845` iter/sec (`stddev: 0.00008127211902349351`)	`1.55`
`tests/benchmarks.py::test_benchmark_tilematrixset_endpoints[/tileMatrixSets/WebMercatorQuad]`	`382.022420299441` iter/sec (`stddev: 0.0015673098671692876`)	`532.7969254762388` iter/sec (`stddev: 0.00021381212034333785`)	`1.39`
`tests/benchmarks.py::test_benchmark_tilematrixset_endpoints[/collections/public.landsat_wrs/tiles/WGS1984Quad]`	`141.9559763943754` iter/sec (`stddev: 0.0019845069146193083`)	`185.9609818775343` iter/sec (`stddev: 0.0004695337976789282`)	`1.31`
`tests/benchmarks.py::test_benchmark_tilematrixset_endpoints[/collections/public.landsat_wrs/tiles/WebMercatorQuad]`	`135.023357229028` iter/sec (`stddev: 0.0018019594767435098`)	`182.4968335261223` iter/sec (`stddev: 0.0005136992803556794`)	`1.35`