From 57c51253e886e8ea6429ac8710b97975c05fd7bb Mon Sep 17 00:00:00 2001 From: Rajendra Kadam Date: Mon, 3 Jun 2024 18:03:22 +0530 Subject: [PATCH 1/4] Add support for '$not' operator --- langchain_postgres/vectorstores.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/langchain_postgres/vectorstores.py b/langchain_postgres/vectorstores.py index 89b78fb..cf46045 100644 --- a/langchain_postgres/vectorstores.py +++ b/langchain_postgres/vectorstores.py @@ -74,7 +74,7 @@ class DistanceStrategy(str, enum.Enum): "$ilike", } -LOGICAL_OPERATORS = {"$and", "$or"} +LOGICAL_OPERATORS = {"$and", "$or", "$not"} SUPPORTED_OPERATORS = ( set(COMPARISONS_TO_NATIVE) @@ -832,21 +832,21 @@ def _create_filter_clause(self, filters: Any) -> Any: """ if isinstance(filters, dict): if len(filters) == 1: - # The only operators allowed at the top level are $AND and $OR + # The only operators allowed at the top level are $AND, $OR, and $NOT # First check if an operator or a field key, value = list(filters.items())[0] if key.startswith("$"): # Then it's an operator - if key.lower() not in ["$and", "$or"]: + if key.lower() not in ["$and", "$or", "$not"]: raise ValueError( - f"Invalid filter condition. Expected $and or $or " + f"Invalid filter condition. Expected $and, $or or $not " f"but got: {key}" ) else: # Then it's a field return self._handle_field_filter(key, filters[key]) - # Here we handle the $and and $or operators + # Here we handle the $and, $or, and $not operators if not isinstance(value, list): raise ValueError( f"Expected a list, but got {type(value)} for value: {value}" @@ -873,9 +873,17 @@ def _create_filter_clause(self, filters: Any) -> Any: "Invalid filter condition. Expected a dictionary " "but got an empty dictionary" ) + elif key.lower() == "$not": + not_conditions = [ + self._create_filter_clause(item) for item in value + ] + not_ = sqlalchemy.and_( + *[sqlalchemy.not_(condition) for condition in not_conditions] + ) + return not_ else: raise ValueError( - f"Invalid filter condition. Expected $and or $or " + f"Invalid filter condition. Expected $and, $or or $not " f"but got: {key}" ) elif len(filters) > 1: From fb66437c6e98987796cd0c86f15b10183aea41e1 Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Mon, 10 Jun 2024 14:37:49 -0400 Subject: [PATCH 2/4] qxqx --- .../unit_tests/fixtures/filtering_test_cases.py | 17 +++++++++++++++++ tests/unit_tests/test_vectorstore.py | 2 ++ 2 files changed, 19 insertions(+) diff --git a/tests/unit_tests/fixtures/filtering_test_cases.py b/tests/unit_tests/fixtures/filtering_test_cases.py index 701260c..a354847 100644 --- a/tests/unit_tests/fixtures/filtering_test_cases.py +++ b/tests/unit_tests/fixtures/filtering_test_cases.py @@ -165,6 +165,23 @@ {"height": {"$lte": 5.8}}, [2, 3], ), + # Test for $not operator + ( + {"$not": {"id": 1}}, + [2, 3], + ), + ( + {"$not": {"name": "adam"}}, + [2, 3], + ), + ( + {"$not": {"is_active": True}}, + [2], + ), + ( + {"$not": {"height": {"$gt": 5.0}}}, + [3], + ), ] TYPE_3_FILTERING_TEST_CASES = [ diff --git a/tests/unit_tests/test_vectorstore.py b/tests/unit_tests/test_vectorstore.py index fcba8ef..70bd825 100644 --- a/tests/unit_tests/test_vectorstore.py +++ b/tests/unit_tests/test_vectorstore.py @@ -992,6 +992,7 @@ async def test_async_pgvector_with_with_metadata_filters_5( {"$eq": {}}, {"$exists": {}}, {"$exists": 1}, + {"$not": 2}, ], ) def test_invalid_filters(pgvector: PGVector, invalid_filter: Any) -> None: @@ -1017,4 +1018,5 @@ def test_validate_operators() -> None: "$ne", "$nin", "$or", + "$not", ] From 3711723d6d1985de418435f1c230024b3cd43749 Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Mon, 10 Jun 2024 14:41:37 -0400 Subject: [PATCH 3/4] x --- tests/unit_tests/test_vectorstore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit_tests/test_vectorstore.py b/tests/unit_tests/test_vectorstore.py index 70bd825..cf0a184 100644 --- a/tests/unit_tests/test_vectorstore.py +++ b/tests/unit_tests/test_vectorstore.py @@ -1017,6 +1017,6 @@ def test_validate_operators() -> None: "$lte", "$ne", "$nin", - "$or", "$not", + "$or", ] From 270a933f1d5f1803add890c2e9cb78b764bf1f5f Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Mon, 10 Jun 2024 14:49:26 -0400 Subject: [PATCH 4/4] qxqx --- langchain_postgres/vectorstores.py | 39 +++++++++----- .../fixtures/filtering_test_cases.py | 52 ++++++++++++------- 2 files changed, 61 insertions(+), 30 deletions(-) diff --git a/langchain_postgres/vectorstores.py b/langchain_postgres/vectorstores.py index 4092e05..5f05aea 100644 --- a/langchain_postgres/vectorstores.py +++ b/langchain_postgres/vectorstores.py @@ -1262,12 +1262,11 @@ def _create_filter_clause(self, filters: Any) -> Any: # Then it's a field return self._handle_field_filter(key, filters[key]) - # Here we handle the $and, $or, and $not operators - if not isinstance(value, list): - raise ValueError( - f"Expected a list, but got {type(value)} for value: {value}" - ) if key.lower() == "$and": + if not isinstance(value, list): + raise ValueError( + f"Expected a list, but got {type(value)} for value: {value}" + ) and_ = [self._create_filter_clause(el) for el in value] if len(and_) > 1: return sqlalchemy.and_(*and_) @@ -1279,6 +1278,10 @@ def _create_filter_clause(self, filters: Any) -> Any: "but got an empty dictionary" ) elif key.lower() == "$or": + if not isinstance(value, list): + raise ValueError( + f"Expected a list, but got {type(value)} for value: {value}" + ) or_ = [self._create_filter_clause(el) for el in value] if len(or_) > 1: return sqlalchemy.or_(*or_) @@ -1290,13 +1293,25 @@ def _create_filter_clause(self, filters: Any) -> Any: "but got an empty dictionary" ) elif key.lower() == "$not": - not_conditions = [ - self._create_filter_clause(item) for item in value - ] - not_ = sqlalchemy.and_( - *[sqlalchemy.not_(condition) for condition in not_conditions] - ) - return not_ + if isinstance(value, list): + not_conditions = [ + self._create_filter_clause(item) for item in value + ] + not_ = sqlalchemy.and_( + *[ + sqlalchemy.not_(condition) + for condition in not_conditions + ] + ) + return not_ + elif isinstance(value, dict): + not_ = self._create_filter_clause(value) + return sqlalchemy.not_(not_) + else: + raise ValueError( + f"Invalid filter condition. Expected a dictionary " + f"or a list but got: {type(value)}" + ) else: raise ValueError( f"Invalid filter condition. Expected $and, $or or $not " diff --git a/tests/unit_tests/fixtures/filtering_test_cases.py b/tests/unit_tests/fixtures/filtering_test_cases.py index a354847..181e8ba 100644 --- a/tests/unit_tests/fixtures/filtering_test_cases.py +++ b/tests/unit_tests/fixtures/filtering_test_cases.py @@ -81,7 +81,7 @@ TYPE_2_FILTERING_TEST_CASES = [ # These involve equality checks and other operators - # like $ne, $gt, $gte, $lt, $lte, $not + # like $ne, $gt, $gte, $lt, $lte ( {"id": 1}, [1], @@ -165,42 +165,58 @@ {"height": {"$lte": 5.8}}, [2, 3], ), +] + +TYPE_3_FILTERING_TEST_CASES = [ + # These involve usage of AND, OR and NOT operators + ( + {"$or": [{"id": 1}, {"id": 2}]}, + [1, 2], + ), + ( + {"$or": [{"id": 1}, {"name": "bob"}]}, + [1, 2], + ), + ( + {"$and": [{"id": 1}, {"id": 2}]}, + [], + ), + ( + {"$or": [{"id": 1}, {"id": 2}, {"id": 3}]}, + [1, 2, 3], + ), # Test for $not operator ( {"$not": {"id": 1}}, [2, 3], ), ( - {"$not": {"name": "adam"}}, + {"$not": [{"id": 1}]}, [2, 3], ), ( - {"$not": {"is_active": True}}, - [2], + {"$not": {"name": "adam"}}, + [2, 3], ), ( - {"$not": {"height": {"$gt": 5.0}}}, - [3], + {"$not": [{"name": "adam"}]}, + [2, 3], ), -] - -TYPE_3_FILTERING_TEST_CASES = [ - # These involve usage of AND and OR operators ( - {"$or": [{"id": 1}, {"id": 2}]}, - [1, 2], + {"$not": {"is_active": True}}, + [2], ), ( - {"$or": [{"id": 1}, {"name": "bob"}]}, - [1, 2], + {"$not": [{"is_active": True}]}, + [2], ), ( - {"$and": [{"id": 1}, {"id": 2}]}, - [], + {"$not": {"height": {"$gt": 5.0}}}, + [3], ), ( - {"$or": [{"id": 1}, {"id": 2}, {"id": 3}]}, - [1, 2, 3], + {"$not": [{"height": {"$gt": 5.0}}]}, + [3], ), ]