From fdb53adaf228b5d3fdd140ade2f3b74db8720f5d Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 7 Jun 2024 21:19:49 +0100 Subject: [PATCH 1/3] add `cast` function --- docs/src/index.rst | 1 + docs/src/piccolo/functions/aggregate.rst | 34 +++++ docs/src/piccolo/functions/basic_usage.rst | 53 +++++++ docs/src/piccolo/functions/index.rst | 12 ++ docs/src/piccolo/functions/string.rst | 40 +++++ .../src/piccolo/functions/type_conversion.rst | 25 ++++ piccolo/columns/base.py | 4 + piccolo/query/functions/__init__.py | 2 + piccolo/query/functions/aggregate.py | 24 +-- piccolo/query/functions/type_conversion.py | 82 ++++++++++ piccolo/querystring.py | 12 ++ tests/columns/test_array.py | 21 +-- tests/query/test_functions.py | 140 +++++++++++++++++- 13 files changed, 426 insertions(+), 24 deletions(-) create mode 100644 docs/src/piccolo/functions/aggregate.rst create mode 100644 docs/src/piccolo/functions/basic_usage.rst create mode 100644 docs/src/piccolo/functions/index.rst create mode 100644 docs/src/piccolo/functions/string.rst create mode 100644 docs/src/piccolo/functions/type_conversion.rst create mode 100644 piccolo/query/functions/type_conversion.py diff --git a/docs/src/index.rst b/docs/src/index.rst index ef956ce49..20b48aa5f 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -16,6 +16,7 @@ batteries included. piccolo/getting_started/index piccolo/query_types/index piccolo/query_clauses/index + piccolo/functions/index piccolo/schema/index piccolo/projects_and_apps/index piccolo/engines/index diff --git a/docs/src/piccolo/functions/aggregate.rst b/docs/src/piccolo/functions/aggregate.rst new file mode 100644 index 000000000..3b1a95a4e --- /dev/null +++ b/docs/src/piccolo/functions/aggregate.rst @@ -0,0 +1,34 @@ +Aggregate functions +=================== + +.. currentmodule:: piccolo.query.functions.aggregate + +Avg +--- + +.. autoclass:: Avg + :class-doc-from: class + +Count +----- + +.. autoclass:: Count + :class-doc-from: class + +Min +--- + +.. autoclass:: Min + :class-doc-from: class + +Max +--- + +.. autoclass:: Max + :class-doc-from: class + +Sum +--- + +.. autoclass:: Sum + :class-doc-from: class diff --git a/docs/src/piccolo/functions/basic_usage.rst b/docs/src/piccolo/functions/basic_usage.rst new file mode 100644 index 000000000..de45883b0 --- /dev/null +++ b/docs/src/piccolo/functions/basic_usage.rst @@ -0,0 +1,53 @@ +Basic Usage +=========== + +Select queries +-------------- + +Functions can be used in ``select`` queries - here's an example, where we +convert the values to uppercase: + +.. code-block:: python + + >>> from piccolo.query.functions import Upper + + >>> await Band.select( + ... Upper(Band.name, alias="name") + ... ) + + [{"name": "PYTHONISTAS"}] + +Where clauses +------------- + +Functions can also be used in ``where`` clauses. + +.. code-block:: python + + >>> from piccolo.query.functions import Length + + >>> await Band.select( + ... Band.name + ... ).where( + ... Length(Band.name) > 10 + ... ) + + [{"name": "Pythonistas"}] + +Update queries +-------------- + +And even in ``update`` queries: + +.. code-block:: python + + >>> from piccolo.query.functions import Upper + + >>> await Band.update( + ... {Band.name: Upper(Band.name)}, + ... force=True + ... ).returning(Band.name) + + [{"name": "PYTHONISTAS"}, {"name": "RUSTACEANS"}, {"name": "C-SHARPS"}] + +Pretty much everywhere. diff --git a/docs/src/piccolo/functions/index.rst b/docs/src/piccolo/functions/index.rst new file mode 100644 index 000000000..d9a412bbc --- /dev/null +++ b/docs/src/piccolo/functions/index.rst @@ -0,0 +1,12 @@ +Functions +========= + +Functions can be used to modify how queries are run, and what is returned. + +.. toctree:: + :maxdepth: 1 + + ./basic_usage + ./string + ./type_conversion + ./aggregate diff --git a/docs/src/piccolo/functions/string.rst b/docs/src/piccolo/functions/string.rst new file mode 100644 index 000000000..dbc09125a --- /dev/null +++ b/docs/src/piccolo/functions/string.rst @@ -0,0 +1,40 @@ +String functions +================ + +.. currentmodule:: piccolo.query.functions.string + +Length +------ + +.. autoclass:: Length + :class-doc-from: class + +Lower +----- + +.. autoclass:: Lower + :class-doc-from: class + +Ltrim +----- + +.. autoclass:: Ltrim + :class-doc-from: class + +Reverse +------- + +.. autoclass:: Reverse + :class-doc-from: class + +Rtrim +----- + +.. autoclass:: Rtrim + :class-doc-from: class + +Upper +----- + +.. autoclass:: Upper + :class-doc-from: class diff --git a/docs/src/piccolo/functions/type_conversion.rst b/docs/src/piccolo/functions/type_conversion.rst new file mode 100644 index 000000000..7e2743d2b --- /dev/null +++ b/docs/src/piccolo/functions/type_conversion.rst @@ -0,0 +1,25 @@ +Type conversion functions +========================= + +Cast +---- + +.. currentmodule:: piccolo.query.functions.type_conversion + +.. autoclass:: Cast + +Notes on databases +------------------ + +Postgres and CockroachDB have very rich type systems, and you can convert +between most types. SQLite is more limited. + +The following query will work in Postgres / Cockroach, but you might get +unexpected results in SQLite, because it doesn't have a native ``TIME`` column +type: + +.. code-block:: python + + >>> from piccolo.columns import Time + >>> from piccolo.query.functions import Cast + >>> await Concert.select(Cast(Concert.starts, Time())) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index ce452260c..2520502d6 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -201,6 +201,10 @@ def table(self) -> t.Type[Table]: ) return self._table + @table.setter + def table(self, value: t.Type[Table]): + self._table = value + ########################################################################### # Used by Foreign Keys: diff --git a/piccolo/query/functions/__init__.py b/piccolo/query/functions/__init__.py index d0195cc40..f7c841d0b 100644 --- a/piccolo/query/functions/__init__.py +++ b/piccolo/query/functions/__init__.py @@ -1,8 +1,10 @@ from .aggregate import Avg, Count, Max, Min, Sum from .string import Length, Lower, Ltrim, Reverse, Rtrim, Upper +from .type_conversion import Cast __all__ = ( "Avg", + "Cast", "Count", "Length", "Lower", diff --git a/piccolo/query/functions/aggregate.py b/piccolo/query/functions/aggregate.py index 61dd36a46..c50e557db 100644 --- a/piccolo/query/functions/aggregate.py +++ b/piccolo/query/functions/aggregate.py @@ -12,17 +12,17 @@ class Avg(Function): .. code-block:: python - await Band.select(Avg(Band.popularity)).run() + await Band.select(Avg(Band.popularity)) # We can use an alias. These two are equivalent: await Band.select( Avg(Band.popularity, alias="popularity_avg") - ).run() + ) await Band.select( Avg(Band.popularity).as_alias("popularity_avg") - ).run() + ) """ @@ -103,17 +103,17 @@ class Min(Function): .. code-block:: python - await Band.select(Min(Band.popularity)).run() + await Band.select(Min(Band.popularity)) # We can use an alias. These two are equivalent: await Band.select( Min(Band.popularity, alias="popularity_min") - ).run() + ) await Band.select( Min(Band.popularity).as_alias("popularity_min") - ).run() + ) """ @@ -128,17 +128,17 @@ class Max(Function): await Band.select( Max(Band.popularity) - ).run() + ) # We can use an alias. These two are equivalent: await Band.select( Max(Band.popularity, alias="popularity_max") - ).run() + ) await Band.select( Max(Band.popularity).as_alias("popularity_max") - ).run() + ) """ @@ -153,17 +153,17 @@ class Sum(Function): await Band.select( Sum(Band.popularity) - ).run() + ) # We can use an alias. These two are equivalent: await Band.select( Sum(Band.popularity, alias="popularity_sum") - ).run() + ) await Band.select( Sum(Band.popularity).as_alias("popularity_sum") - ).run() + ) """ diff --git a/piccolo/query/functions/type_conversion.py b/piccolo/query/functions/type_conversion.py new file mode 100644 index 000000000..1f613565b --- /dev/null +++ b/piccolo/query/functions/type_conversion.py @@ -0,0 +1,82 @@ +import typing as t + +from piccolo.columns.base import Column +from piccolo.querystring import QueryString + + +class Cast(QueryString): + def __init__( + self, + identifier: t.Union[Column, QueryString], + as_type: Column, + alias: t.Optional[str] = None, + ): + """ + Cast a value to a different type. For example:: + + >>> class piccolo.query.functions import Cast + + >>> await Concert.select( + ... Cast(Concert.starts, Time(), "start_time") + ... ) + [{"start_time": datetime.time(19, 0)}] + + :param identifier: + Identifies what is being converted (e.g. a column). + :param as_type: + The type to be converted to. + + """ + # Make sure the identifier is a supported type. + + if not isinstance(identifier, (Column, QueryString)): + raise ValueError( + "The identifier is an unsupported type - only Column and " + "QueryString instances are allowed." + ) + + ####################################################################### + # Convert `as_type` to a string which can be used in the query. + + if not isinstance(as_type, Column): + raise ValueError("The `as_type` value must be a Column instance.") + + # We need to give the column a reference to a table, and hence + # the database engine, as the column type is sometimes dependent + # on which database is being used. + from piccolo.table import Table, create_table_class + + table: t.Optional[t.Type[Table]] = None + + if isinstance(identifier, Column): + table = identifier._meta.table + elif isinstance(identifier, QueryString): + table = ( + identifier.columns[0]._meta.table + if identifier.columns + else None + ) + + as_type._meta.table = table or create_table_class("Table") + as_type_string = as_type.column_type + + ####################################################################### + # Preserve the original alias from the column. + + if isinstance(identifier, Column): + alias = ( + alias + or identifier._alias + or identifier._meta.get_default_alias() + ) + + ####################################################################### + + super().__init__( + f"CAST({{}} AS {as_type_string})", + identifier, + alias=alias, + ) + + +__all__ = ("Cast",) diff --git a/piccolo/querystring.py b/piccolo/querystring.py index 7f3f3e42a..c2fc0f80f 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -270,6 +270,18 @@ def __add__(self, value) -> QueryString: def __sub__(self, value) -> QueryString: return QueryString("{} - {}", self, value) + def __gt__(self, value) -> QueryString: + return QueryString("{} > {}", self, value) + + def __ge__(self, value) -> QueryString: + return QueryString("{} >= {}", self, value) + + def __lt__(self, value) -> QueryString: + return QueryString("{} < {}", self, value) + + def __le__(self, value) -> QueryString: + return QueryString("{} <= {}", self, value) + def is_in(self, value) -> QueryString: return QueryString("{} IN {}", self, value) diff --git a/tests/columns/test_array.py b/tests/columns/test_array.py index 4677ef995..7da4dadfb 100644 --- a/tests/columns/test_array.py +++ b/tests/columns/test_array.py @@ -11,7 +11,7 @@ Timestamptz, ) from piccolo.table import Table -from tests.base import engines_only, sqlite_only +from tests.base import engines_only, engines_skip, sqlite_only class MyTable(Table): @@ -40,7 +40,6 @@ def setUp(self): def tearDown(self): MyTable.alter().drop_table().run_sync() - @engines_only("postgres", "sqlite") def test_storage(self): """ Make sure data can be stored and retrieved. @@ -54,7 +53,7 @@ def test_storage(self): assert row is not None self.assertEqual(row.value, [1, 2, 3]) - @engines_only("postgres") + @engines_skip("sqlite") def test_index(self): """ Indexes should allow individual array elements to be queried. @@ -68,7 +67,7 @@ def test_index(self): MyTable.select(MyTable.value[0]).first().run_sync(), {"value": 1} ) - @engines_only("postgres") + @engines_skip("sqlite") def test_all(self): """ Make sure rows can be retrieved where all items in an array match a @@ -95,7 +94,7 @@ def test_all(self): None, ) - @engines_only("postgres") + @engines_skip("sqlite") def test_any(self): """ Make sure rows can be retrieved where any items in an array match a @@ -122,7 +121,7 @@ def test_any(self): None, ) - @engines_only("postgres") + @engines_skip("sqlite") def test_cat(self): """ Make sure values can be appended to an array. @@ -137,7 +136,8 @@ def test_cat(self): ).run_sync() self.assertEqual( - MyTable.select().run_sync(), [{"id": 1, "value": [1, 1, 1, 2]}] + MyTable.select(MyTable.value).run_sync(), + [{"value": [1, 1, 1, 2]}], ) # Try plus symbol @@ -147,7 +147,8 @@ def test_cat(self): ).run_sync() self.assertEqual( - MyTable.select().run_sync(), [{"id": 1, "value": [1, 1, 1, 2, 3]}] + MyTable.select(MyTable.value).run_sync(), + [{"value": [1, 1, 1, 2, 3]}], ) # Make sure non-list values work @@ -157,8 +158,8 @@ def test_cat(self): ).run_sync() self.assertEqual( - MyTable.select().run_sync(), - [{"id": 1, "value": [1, 1, 1, 2, 3, 4]}], + MyTable.select(MyTable.value).run_sync(), + [{"value": [1, 1, 1, 2, 3, 4]}], ) @sqlite_only diff --git a/tests/query/test_functions.py b/tests/query/test_functions.py index abe9a5f01..a970d9fa1 100644 --- a/tests/query/test_functions.py +++ b/tests/query/test_functions.py @@ -1,6 +1,7 @@ from unittest import TestCase -from piccolo.query.functions.string import Reverse, Upper +from piccolo.columns import Integer, Text, Varchar +from piccolo.query.functions import Cast, Length, Reverse, Upper from piccolo.querystring import QueryString from piccolo.table import create_db_tables_sync, drop_db_tables_sync from tests.base import engines_skip @@ -16,7 +17,13 @@ def setUp(self) -> None: manager = Manager({Manager.name: "Guido"}) manager.save().run_sync() - band = Band({Band.name: "Pythonistas", Band.manager: manager}) + band = Band( + { + Band.name: "Pythonistas", + Band.manager: manager, + Band.popularity: 1000, + } + ) band.save().run_sync() def tearDown(self) -> None: @@ -100,3 +107,132 @@ def test_where_with_joined_column(self): .run_sync() ) self.assertListEqual(response, [{"name": "Pythonistas"}]) + + +class TestCast(FunctionTest): + def test_varchar(self): + """ + Make sure that casting to ``Varchar`` works. + """ + response = Band.select( + Cast( + Band.popularity, + as_type=Varchar(), + ) + ).run_sync() + + self.assertListEqual( + response, + [{"popularity": "1000"}], + ) + + def test_text(self): + """ + Make sure that casting to ``Text`` works. + """ + response = Band.select( + Cast( + Band.popularity, + as_type=Text(), + ) + ).run_sync() + + self.assertListEqual( + response, + [{"popularity": "1000"}], + ) + + def test_integer(self): + """ + Make sure that casting to ``Integer`` works. + """ + Band.update({Band.name: "1111"}, force=True).run_sync() + + response = Band.select( + Cast( + Band.name, + as_type=Integer(), + ) + ).run_sync() + + self.assertListEqual( + response, + [{"name": 1111}], + ) + + def test_join(self): + """ + Make sure that casting works with joins. + """ + Manager.update({Manager.name: "1111"}, force=True).run_sync() + + response = Band.select( + Band.name, + Cast( + Band.manager.name, + as_type=Integer(), + ), + ).run_sync() + + self.assertListEqual( + response, + [ + { + "name": "Pythonistas", + "manager.name": 1111, + } + ], + ) + + def test_nested_inner(self): + """ + Make sure ``Cast`` can be passed into other functions. + """ + Band.update({Band.name: "1111"}, force=True).run_sync() + + response = Band.select( + Length( + Cast( + Band.popularity, + as_type=Varchar(), + ) + ) + ).run_sync() + + self.assertListEqual( + response, + [{"length": 4}], + ) + + def test_nested_outer(self): + """ + Make sure a querystring can be passed into ``Cast`` (meaning it can be + nested). + """ + response = Band.select( + Cast( + Length(Band.name), + as_type=Varchar(), + alias="length", + ) + ).run_sync() + + self.assertListEqual( + response, + [{"length": str(len("Pythonistas"))}], + ) + + def test_where_clause(self): + """ + Make sure ``Cast`` works in a where clause. + """ + response = ( + Band.select(Band.name, Band.popularity) + .where(Cast(Band.popularity, Varchar()) == "1000") + .run_sync() + ) + + self.assertListEqual( + response, + [{"name": "Pythonistas", "popularity": 1000}], + ) From c1e89afe9c2ea69eb70eafe28b76fcda2819e001 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 7 Jun 2024 21:22:25 +0100 Subject: [PATCH 2/3] fix typo --- piccolo/query/functions/type_conversion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piccolo/query/functions/type_conversion.py b/piccolo/query/functions/type_conversion.py index 1f613565b..e8f1dfee5 100644 --- a/piccolo/query/functions/type_conversion.py +++ b/piccolo/query/functions/type_conversion.py @@ -14,7 +14,7 @@ def __init__( """ Cast a value to a different type. For example:: - >>> class piccolo.query.functions import Cast + >>> from piccolo.query.functions import Cast >>> await Concert.select( ... Cast(Concert.starts, Time(), "start_time") From c84d704f6432807b61c27e95b114a65af583121b Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 7 Jun 2024 21:35:17 +0100 Subject: [PATCH 3/3] revert accidental `test_array.py` changes they weren't meant to be in this branch --- tests/columns/test_array.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tests/columns/test_array.py b/tests/columns/test_array.py index 7da4dadfb..4677ef995 100644 --- a/tests/columns/test_array.py +++ b/tests/columns/test_array.py @@ -11,7 +11,7 @@ Timestamptz, ) from piccolo.table import Table -from tests.base import engines_only, engines_skip, sqlite_only +from tests.base import engines_only, sqlite_only class MyTable(Table): @@ -40,6 +40,7 @@ def setUp(self): def tearDown(self): MyTable.alter().drop_table().run_sync() + @engines_only("postgres", "sqlite") def test_storage(self): """ Make sure data can be stored and retrieved. @@ -53,7 +54,7 @@ def test_storage(self): assert row is not None self.assertEqual(row.value, [1, 2, 3]) - @engines_skip("sqlite") + @engines_only("postgres") def test_index(self): """ Indexes should allow individual array elements to be queried. @@ -67,7 +68,7 @@ def test_index(self): MyTable.select(MyTable.value[0]).first().run_sync(), {"value": 1} ) - @engines_skip("sqlite") + @engines_only("postgres") def test_all(self): """ Make sure rows can be retrieved where all items in an array match a @@ -94,7 +95,7 @@ def test_all(self): None, ) - @engines_skip("sqlite") + @engines_only("postgres") def test_any(self): """ Make sure rows can be retrieved where any items in an array match a @@ -121,7 +122,7 @@ def test_any(self): None, ) - @engines_skip("sqlite") + @engines_only("postgres") def test_cat(self): """ Make sure values can be appended to an array. @@ -136,8 +137,7 @@ def test_cat(self): ).run_sync() self.assertEqual( - MyTable.select(MyTable.value).run_sync(), - [{"value": [1, 1, 1, 2]}], + MyTable.select().run_sync(), [{"id": 1, "value": [1, 1, 1, 2]}] ) # Try plus symbol @@ -147,8 +147,7 @@ def test_cat(self): ).run_sync() self.assertEqual( - MyTable.select(MyTable.value).run_sync(), - [{"value": [1, 1, 1, 2, 3]}], + MyTable.select().run_sync(), [{"id": 1, "value": [1, 1, 1, 2, 3]}] ) # Make sure non-list values work @@ -158,8 +157,8 @@ def test_cat(self): ).run_sync() self.assertEqual( - MyTable.select(MyTable.value).run_sync(), - [{"value": [1, 1, 1, 2, 3, 4]}], + MyTable.select().run_sync(), + [{"id": 1, "value": [1, 1, 1, 2, 3, 4]}], ) @sqlite_only