From b850d4a1d40b66d06e0d9691a14a5dfcf1f1fdbf Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 24 May 2024 19:54:03 +0100 Subject: [PATCH 01/39] support upper and lower --- piccolo/columns/base.py | 24 +++++++++++++++++---- piccolo/columns/column_types.py | 38 +++++++++++++++++++++++++++++++++ piccolo/query/methods/select.py | 34 +++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 4 deletions(-) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 886a0ee48..91406a6e7 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -272,7 +272,10 @@ def _get_path(self, include_quotes: bool = False): return f"{self.table._meta.tablename}.{column_name}" def get_full_name( - self, with_alias: bool = True, include_quotes: bool = True + self, + with_alias: bool = True, + include_quotes: bool = True, + operator: str | None = None, ) -> str: """ Returns the full column name, taking into account joins. @@ -306,6 +309,9 @@ def get_full_name( """ full_name = self._get_path(include_quotes=include_quotes) + if operator: + full_name = f"{operator}({full_name})" + if with_alias and self.call_chain: alias = self.get_default_alias() if include_quotes: @@ -827,18 +833,25 @@ def get_select_string( How to refer to this column in a SQL query, taking account of any joins and aliases. """ + operator = getattr(self, "operator", None) + if with_alias: if self._alias: original_name = self._meta.get_full_name( with_alias=False, + operator=operator, ) return f'{original_name} AS "{self._alias}"' else: return self._meta.get_full_name( with_alias=True, + operator=operator, ) - return self._meta.get_full_name(with_alias=False) + return self._meta.get_full_name( + with_alias=False, + operator=operator, + ) def get_where_string(self, engine_type: str) -> str: return self.get_select_string( @@ -945,8 +958,8 @@ def ddl(self) -> str: return query - def copy(self) -> Column: - column: Column = copy.copy(self) + def copy(self: Self) -> Self: + column = copy.copy(self) column._meta = self._meta.copy() return column @@ -971,3 +984,6 @@ def __repr__(self): f"{table_class_name}.{self._meta.name} - " f"{self.__class__.__name__}" ) + + +Self = t.TypeVar("Self", bound=Column) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 2afcfb741..c3e1e3e1d 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -287,6 +287,14 @@ def get_querystring( ############################################################################### +def Upper(column: Varchar) -> Varchar: + return column.upper() + + +def Lower(column: Varchar) -> Varchar: + return column.lower() + + class Varchar(Column): """ Used for storing text when you want to enforce character length limits. @@ -313,6 +321,7 @@ class Band(Table): value_type = str concat_delegate: ConcatDelegate = ConcatDelegate() + operator: t.Literal["upper", "lower"] | None = None def __init__( self, @@ -324,6 +333,7 @@ def __init__( self.length = length self.default = default + self.operator = None kwargs.update({"length": length, "default": default}) super().__init__(**kwargs) @@ -346,6 +356,19 @@ def __radd__(self, value: t.Union[str, Varchar, Text]) -> QueryString: reverse=True, ) + ########################################################################### + # Operators + + def upper(self) -> Varchar: + column = self.copy() + column.operator = "upper" + return column + + def lower(self) -> Varchar: + column = self.copy() + column.operator = "lower" + return column + ########################################################################### # Descriptors @@ -422,6 +445,7 @@ class Band(Table): value_type = str concat_delegate: ConcatDelegate = ConcatDelegate() + operator: t.Literal["upper", "lower"] | None = None def __init__( self, @@ -430,6 +454,7 @@ def __init__( ) -> None: self._validate_default(default, (str, None)) self.default = default + self.operator = None kwargs.update({"default": default}) super().__init__(**kwargs) @@ -448,6 +473,19 @@ def __radd__(self, value: t.Union[str, Varchar, Text]) -> QueryString: reverse=True, ) + ########################################################################### + # Operators + + def upper(self) -> Text: + column = self.copy() + column.operator = "upper" + return column + + def lower(self) -> Text: + column = self.copy() + column.operator = "lower" + return column + ########################################################################### # Descriptors diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index a2a77b155..f39a7425f 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -41,6 +41,10 @@ def is_numeric_column(column: Column) -> bool: return column.value_type in (int, decimal.Decimal, float) +def is_string_column(column: Column) -> bool: + return column.value_type is str + + class SelectRaw(Selectable): def __init__(self, sql: str, *args: t.Any) -> None: """ @@ -279,6 +283,36 @@ def get_select_string( return f'SUM({column_name}) AS "{self._alias}"' +class Upper(Selectable): + """ + ``UPPER()`` SQL function. The column type must contain text to run the + query. + + .. code-block:: python + + >>> await Band.select( + ... Upper(Band.name) + ... ).run() + [{"name": "PYTHONISTAS"}] + + """ + + def __init__(self, column: Column, alias: str = "upper"): + if is_string_column(column): + self.column = column + else: + raise ValueError( + "Column type must contain a string to run the query." + ) + self._alias = alias + + def get_select_string( + self, engine_type: str, with_alias: bool = True + ) -> str: + column_name = self.column._meta.get_full_name(with_alias=False) + return f'SUM({column_name}) AS "{self._alias}"' + + OptionalDict = t.Optional[t.Dict[str, t.Any]] From f772a37c48a8668ac7c6074318c8d70bcc492ea3 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 24 May 2024 19:55:40 +0100 Subject: [PATCH 02/39] remove prototype code --- piccolo/query/methods/select.py | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index f39a7425f..31c38fc6a 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -283,36 +283,6 @@ def get_select_string( return f'SUM({column_name}) AS "{self._alias}"' -class Upper(Selectable): - """ - ``UPPER()`` SQL function. The column type must contain text to run the - query. - - .. code-block:: python - - >>> await Band.select( - ... Upper(Band.name) - ... ).run() - [{"name": "PYTHONISTAS"}] - - """ - - def __init__(self, column: Column, alias: str = "upper"): - if is_string_column(column): - self.column = column - else: - raise ValueError( - "Column type must contain a string to run the query." - ) - self._alias = alias - - def get_select_string( - self, engine_type: str, with_alias: bool = True - ) -> str: - column_name = self.column._meta.get_full_name(with_alias=False) - return f'SUM({column_name}) AS "{self._alias}"' - - OptionalDict = t.Optional[t.Dict[str, t.Any]] From cf8fa7b40b7305e5c6f75f45b327b61538943385 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 24 May 2024 21:10:13 +0100 Subject: [PATCH 03/39] rename `operator` to `outer_function` --- piccolo/columns/base.py | 22 +++++++++++++++------- piccolo/columns/column_types.py | 27 +++++++++++++++------------ 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 91406a6e7..c406fddd1 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -275,7 +275,7 @@ def get_full_name( self, with_alias: bool = True, include_quotes: bool = True, - operator: str | None = None, + outer_function: str | None = None, ) -> str: """ Returns the full column name, taking into account joins. @@ -305,12 +305,20 @@ def get_full_name( >>> column._meta.get_full_name(include_quotes=False) 'my_table_name.my_column_name' + :param outer_function: + If provided, the column name will be wrapped with the given + function. For example: + + .. code-block python:: + + >>> Band.manager.get_full_name(outer_function="upper") + 'upper(band$manager.name) AS "manager$name"' """ full_name = self._get_path(include_quotes=include_quotes) - if operator: - full_name = f"{operator}({full_name})" + if outer_function: + full_name = f"{outer_function}({full_name})" if with_alias and self.call_chain: alias = self.get_default_alias() @@ -833,24 +841,24 @@ def get_select_string( How to refer to this column in a SQL query, taking account of any joins and aliases. """ - operator = getattr(self, "operator", None) + outer_function = getattr(self, "outer_function", None) if with_alias: if self._alias: original_name = self._meta.get_full_name( with_alias=False, - operator=operator, + outer_function=outer_function, ) return f'{original_name} AS "{self._alias}"' else: return self._meta.get_full_name( with_alias=True, - operator=operator, + outer_function=outer_function, ) return self._meta.get_full_name( with_alias=False, - operator=operator, + outer_function=outer_function, ) def get_where_string(self, engine_type: str) -> str: diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index c3e1e3e1d..d4d593d48 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -287,11 +287,14 @@ def get_querystring( ############################################################################### -def Upper(column: Varchar) -> Varchar: +StringColumn = t.TypeVar("StringColumn", "Varchar", "Text") + + +def Upper(column: StringColumn) -> StringColumn: return column.upper() -def Lower(column: Varchar) -> Varchar: +def Lower(column: StringColumn) -> StringColumn: return column.lower() @@ -321,7 +324,7 @@ class Band(Table): value_type = str concat_delegate: ConcatDelegate = ConcatDelegate() - operator: t.Literal["upper", "lower"] | None = None + outer_function: t.Literal["upper", "lower"] | None = None def __init__( self, @@ -333,7 +336,7 @@ def __init__( self.length = length self.default = default - self.operator = None + self.outer_function = None kwargs.update({"length": length, "default": default}) super().__init__(**kwargs) @@ -357,16 +360,16 @@ def __radd__(self, value: t.Union[str, Varchar, Text]) -> QueryString: ) ########################################################################### - # Operators + # Outer functions def upper(self) -> Varchar: column = self.copy() - column.operator = "upper" + column.outer_function = "upper" return column def lower(self) -> Varchar: column = self.copy() - column.operator = "lower" + column.outer_function = "lower" return column ########################################################################### @@ -445,7 +448,7 @@ class Band(Table): value_type = str concat_delegate: ConcatDelegate = ConcatDelegate() - operator: t.Literal["upper", "lower"] | None = None + outer_function: t.Literal["upper", "lower"] | None = None def __init__( self, @@ -454,7 +457,7 @@ def __init__( ) -> None: self._validate_default(default, (str, None)) self.default = default - self.operator = None + self.outer_function = None kwargs.update({"default": default}) super().__init__(**kwargs) @@ -474,16 +477,16 @@ def __radd__(self, value: t.Union[str, Varchar, Text]) -> QueryString: ) ########################################################################### - # Operators + # Outer functions def upper(self) -> Text: column = self.copy() - column.operator = "upper" + column.outer_function = "upper" return column def lower(self) -> Text: column = self.copy() - column.operator = "lower" + column.outer_function = "lower" return column ########################################################################### From ff452275d8ff294c909283507aca5daafd99c153 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 24 May 2024 21:19:15 +0100 Subject: [PATCH 04/39] remove check for `call_chain` --- piccolo/columns/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index c406fddd1..abed6cb16 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -320,7 +320,7 @@ def get_full_name( if outer_function: full_name = f"{outer_function}({full_name})" - if with_alias and self.call_chain: + if with_alias: alias = self.get_default_alias() if include_quotes: full_name += f' AS "{alias}"' From 9966769be314c7e829ab26226cf35ead16031b75 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 24 May 2024 22:27:12 +0100 Subject: [PATCH 05/39] allow multiple outer_functions to be passed in --- piccolo/columns/base.py | 19 ++++++++-------- piccolo/columns/column_types.py | 40 +++++++++++++++++++++++---------- 2 files changed, 38 insertions(+), 21 deletions(-) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index abed6cb16..10953b0de 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -275,7 +275,7 @@ def get_full_name( self, with_alias: bool = True, include_quotes: bool = True, - outer_function: str | None = None, + outer_functions: t.List[str] | None = None, ) -> str: """ Returns the full column name, taking into account joins. @@ -305,20 +305,21 @@ def get_full_name( >>> column._meta.get_full_name(include_quotes=False) 'my_table_name.my_column_name' - :param outer_function: + :param outer_functions: If provided, the column name will be wrapped with the given function. For example: .. code-block python:: - >>> Band.manager.get_full_name(outer_function="upper") + >>> Band.manager.get_full_name(outer_functions="upper") 'upper(band$manager.name) AS "manager$name"' """ full_name = self._get_path(include_quotes=include_quotes) - if outer_function: - full_name = f"{outer_function}({full_name})" + if outer_functions: + for outer_function in outer_functions: + full_name = f"{outer_function}({full_name})" if with_alias: alias = self.get_default_alias() @@ -841,24 +842,24 @@ def get_select_string( How to refer to this column in a SQL query, taking account of any joins and aliases. """ - outer_function = getattr(self, "outer_function", None) + outer_functions = getattr(self, "outer_functions", None) if with_alias: if self._alias: original_name = self._meta.get_full_name( with_alias=False, - outer_function=outer_function, + outer_functions=outer_functions, ) return f'{original_name} AS "{self._alias}"' else: return self._meta.get_full_name( with_alias=True, - outer_function=outer_function, + outer_functions=outer_functions, ) return self._meta.get_full_name( with_alias=False, - outer_function=outer_function, + outer_functions=outer_functions, ) def get_where_string(self, engine_type: str) -> str: diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index d4d593d48..31440df60 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -324,7 +324,7 @@ class Band(Table): value_type = str concat_delegate: ConcatDelegate = ConcatDelegate() - outer_function: t.Literal["upper", "lower"] | None = None + outer_functions: t.List[t.Literal["upper", "lower"]] def __init__( self, @@ -336,7 +336,7 @@ def __init__( self.length = length self.default = default - self.outer_function = None + self.outer_functions = [] kwargs.update({"length": length, "default": default}) super().__init__(**kwargs) @@ -362,14 +362,22 @@ def __radd__(self, value: t.Union[str, Varchar, Text]) -> QueryString: ########################################################################### # Outer functions - def upper(self) -> Varchar: + def _add_outer_function(self, outer_function) -> Varchar: column = self.copy() - column.outer_function = "upper" + column.outer_functions.append(outer_function) return column + def upper(self) -> Varchar: + return self._add_outer_function("upper") + def lower(self) -> Varchar: - column = self.copy() - column.outer_function = "lower" + return self._add_outer_function("lower") + + ########################################################################### + + def copy(self): + column = super().copy() + column.outer_functions = [*column.outer_functions] return column ########################################################################### @@ -448,7 +456,7 @@ class Band(Table): value_type = str concat_delegate: ConcatDelegate = ConcatDelegate() - outer_function: t.Literal["upper", "lower"] | None = None + outer_functions: t.List[t.Literal["upper", "lower"]] def __init__( self, @@ -457,7 +465,7 @@ def __init__( ) -> None: self._validate_default(default, (str, None)) self.default = default - self.outer_function = None + self.outer_functions = [] kwargs.update({"default": default}) super().__init__(**kwargs) @@ -479,14 +487,22 @@ def __radd__(self, value: t.Union[str, Varchar, Text]) -> QueryString: ########################################################################### # Outer functions - def upper(self) -> Text: + def _add_outer_function(self, outer_function) -> Text: column = self.copy() - column.outer_function = "upper" + column.outer_functions.append(outer_function) return column + def upper(self) -> Text: + return self._add_outer_function("upper") + def lower(self) -> Text: - column = self.copy() - column.outer_function = "lower" + return self._add_outer_function("lower") + + ########################################################################### + + def copy(self): + column = super().copy() + column.outer_functions = [*column.outer_functions] return column ########################################################################### From 8b155b5f06b97ce8fb10fb17b86480eda138d95c Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Fri, 24 May 2024 22:40:51 +0100 Subject: [PATCH 06/39] update docstring --- piccolo/columns/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 10953b0de..6d35ffbc1 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -307,11 +307,11 @@ def get_full_name( :param outer_functions: If provided, the column name will be wrapped with the given - function. For example: + functions. For example: .. code-block python:: - >>> Band.manager.get_full_name(outer_functions="upper") + >>> Band.manager.get_full_name(outer_functions=["upper"]) 'upper(band$manager.name) AS "manager$name"' """ From 9c9514aca0102d5c2ca3a6522cdcbd238fb3e722 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 28 May 2024 15:49:30 +0100 Subject: [PATCH 07/39] use querystrings instead --- piccolo/columns/base.py | 64 ++++--------------- piccolo/columns/column_types.py | 106 +++++++++++++++----------------- piccolo/columns/m2m.py | 22 +++++-- piccolo/columns/readable.py | 16 ++--- piccolo/query/methods/select.py | 29 ++++----- piccolo/query/mixins.py | 7 ++- piccolo/querystring.py | 50 ++++++++++++++- piccolo/table.py | 2 +- 8 files changed, 158 insertions(+), 138 deletions(-) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 6d35ffbc1..50ff9b1a0 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -6,7 +6,6 @@ import inspect import typing as t import uuid -from abc import ABCMeta, abstractmethod from dataclasses import dataclass, field, fields from enum import Enum @@ -32,6 +31,7 @@ NotLike, ) from piccolo.columns.reference import LazyTableReference +from piccolo.querystring import QueryString, Selectable from piccolo.utils.warnings import colored_warning if t.TYPE_CHECKING: # pragma: no cover @@ -275,7 +275,6 @@ def get_full_name( self, with_alias: bool = True, include_quotes: bool = True, - outer_functions: t.List[str] | None = None, ) -> str: """ Returns the full column name, taking into account joins. @@ -305,22 +304,9 @@ def get_full_name( >>> column._meta.get_full_name(include_quotes=False) 'my_table_name.my_column_name' - :param outer_functions: - If provided, the column name will be wrapped with the given - functions. For example: - - .. code-block python:: - - >>> Band.manager.get_full_name(outer_functions=["upper"]) - 'upper(band$manager.name) AS "manager$name"' - """ full_name = self._get_path(include_quotes=include_quotes) - if outer_functions: - for outer_function in outer_functions: - full_name = f"{outer_function}({full_name})" - if with_alias: alias = self.get_default_alias() if include_quotes: @@ -361,32 +347,6 @@ def __deepcopy__(self, memo) -> ColumnMeta: return self.copy() -class Selectable(metaclass=ABCMeta): - """ - Anything which inherits from this can be used in a select query. - """ - - _alias: t.Optional[str] - - @abstractmethod - def get_select_string( - self, engine_type: str, with_alias: bool = True - ) -> str: - """ - In a query, what to output after the select statement - could be a - column name, a sub query, a function etc. For a column it will be the - column name. - """ - raise NotImplementedError() - - def as_alias(self, alias: str) -> Selectable: - """ - Allows column names to be changed in the result of a select. - """ - self._alias = alias - return self - - class Column(Selectable): """ All other columns inherit from ``Column``. Don't use it directly. @@ -837,32 +797,32 @@ def get_default_value(self) -> t.Any: def get_select_string( self, engine_type: str, with_alias: bool = True - ) -> str: + ) -> QueryString: """ How to refer to this column in a SQL query, taking account of any joins and aliases. """ - outer_functions = getattr(self, "outer_functions", None) if with_alias: if self._alias: original_name = self._meta.get_full_name( with_alias=False, - outer_functions=outer_functions, ) - return f'{original_name} AS "{self._alias}"' + return QueryString(f'{original_name} AS "{self._alias}"') else: - return self._meta.get_full_name( - with_alias=True, - outer_functions=outer_functions, + return QueryString( + self._meta.get_full_name( + with_alias=True, + ) ) - return self._meta.get_full_name( - with_alias=False, - outer_functions=outer_functions, + return QueryString( + self._meta.get_full_name( + with_alias=False, + ) ) - def get_where_string(self, engine_type: str) -> str: + def get_where_string(self, engine_type: str) -> QueryString: return self.get_select_string( engine_type=engine_type, with_alias=False ) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 31440df60..3939ec21b 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -60,7 +60,7 @@ class Band(Table): from piccolo.columns.operators.comparison import ArrayAll, ArrayAny from piccolo.columns.operators.string import Concat from piccolo.columns.reference import LazyTableReference -from piccolo.querystring import QueryString, Unquoted +from piccolo.querystring import QueryString, Selectable, Unquoted from piccolo.utils.encoding import dump_json from piccolo.utils.warnings import colored_warning @@ -290,12 +290,51 @@ def get_querystring( StringColumn = t.TypeVar("StringColumn", "Varchar", "Text") -def Upper(column: StringColumn) -> StringColumn: - return column.upper() +class Function(QueryString): + function_name: str + columns: t.List[Column] + + def __init__( + self, + identifier: t.Union[StringColumn, QueryString, str], + alias: t.Optional[str] = None, + ): + self._alias = alias + + if isinstance(identifier, Column): + if not alias: + self._alias = identifier._meta.get_default_alias() + + # We track any columns just in case we need to perform joins + self.columns = [identifier, *getattr(self, "columns", [])] + + column_full_name = identifier._meta.get_full_name(with_alias=False) + super().__init__(f"{self.function_name}({column_full_name})") + elif isinstance(identifier, QueryString): + if identifier._alias: + self._alias = identifier._alias + + super().__init__(f"{self.function_name}({{}})", identifier) + elif isinstance(identifier, str): + super().__init__(f"{self.function_name}({{}})", identifier) + else: + raise ValueError("Unrecognised function type") + + def get_select_string( + self, engine_type: str, with_alias: bool = True + ) -> QueryString: + if with_alias and self._alias: + return QueryString("{} AS " + self._alias, self) + else: + return self -def Lower(column: StringColumn) -> StringColumn: - return column.lower() +class Upper(Function): + function_name = "UPPER" + + +class Lower(Function): + function_name = "LOWER" class Varchar(Column): @@ -324,7 +363,6 @@ class Band(Table): value_type = str concat_delegate: ConcatDelegate = ConcatDelegate() - outer_functions: t.List[t.Literal["upper", "lower"]] def __init__( self, @@ -336,7 +374,6 @@ def __init__( self.length = length self.default = default - self.outer_functions = [] kwargs.update({"length": length, "default": default}) super().__init__(**kwargs) @@ -359,27 +396,6 @@ def __radd__(self, value: t.Union[str, Varchar, Text]) -> QueryString: reverse=True, ) - ########################################################################### - # Outer functions - - def _add_outer_function(self, outer_function) -> Varchar: - column = self.copy() - column.outer_functions.append(outer_function) - return column - - def upper(self) -> Varchar: - return self._add_outer_function("upper") - - def lower(self) -> Varchar: - return self._add_outer_function("lower") - - ########################################################################### - - def copy(self): - column = super().copy() - column.outer_functions = [*column.outer_functions] - return column - ########################################################################### # Descriptors @@ -456,7 +472,6 @@ class Band(Table): value_type = str concat_delegate: ConcatDelegate = ConcatDelegate() - outer_functions: t.List[t.Literal["upper", "lower"]] def __init__( self, @@ -465,7 +480,6 @@ def __init__( ) -> None: self._validate_default(default, (str, None)) self.default = default - self.outer_functions = [] kwargs.update({"default": default}) super().__init__(**kwargs) @@ -484,27 +498,6 @@ def __radd__(self, value: t.Union[str, Varchar, Text]) -> QueryString: reverse=True, ) - ########################################################################### - # Outer functions - - def _add_outer_function(self, outer_function) -> Text: - column = self.copy() - column.outer_functions.append(outer_function) - return column - - def upper(self) -> Text: - return self._add_outer_function("upper") - - def lower(self) -> Text: - return self._add_outer_function("lower") - - ########################################################################### - - def copy(self): - column = super().copy() - column.outer_functions = [*column.outer_functions] - return column - ########################################################################### # Descriptors @@ -2251,6 +2244,7 @@ def __getattribute__(self, name: str) -> t.Union[Column, t.Any]: column_meta: ColumnMeta = object.__getattribute__(self, "_meta") new_column._meta.call_chain = column_meta.call_chain.copy() + new_column._meta.call_chain.append(self) return new_column else: @@ -2368,7 +2362,7 @@ def arrow(self, key: str) -> JSONB: def get_select_string( self, engine_type: str, with_alias: bool = True - ) -> str: + ) -> QueryString: select_string = self._meta.get_full_name(with_alias=False) if self.json_operator is not None: @@ -2378,7 +2372,7 @@ def get_select_string( alias = self._alias or self._meta.get_default_alias() select_string += f' AS "{alias}"' - return select_string + return QueryString(select_string) def eq(self, value) -> Where: """ @@ -2673,7 +2667,9 @@ def __getitem__(self, value: int) -> Array: else: raise ValueError("Only integers can be used for indexing.") - def get_select_string(self, engine_type: str, with_alias=True) -> str: + def get_select_string( + self, engine_type: str, with_alias=True + ) -> QueryString: select_string = self._meta.get_full_name(with_alias=False) if isinstance(self.index, int): @@ -2683,7 +2679,7 @@ def get_select_string(self, engine_type: str, with_alias=True) -> str: alias = self._alias or self._meta.get_default_alias() select_string += f' AS "{alias}"' - return select_string + return QueryString(select_string) def any(self, value: t.Any) -> Where: """ diff --git a/piccolo/columns/m2m.py b/piccolo/columns/m2m.py index 0eefd22e7..90469fc1f 100644 --- a/piccolo/columns/m2m.py +++ b/piccolo/columns/m2m.py @@ -4,7 +4,6 @@ import typing as t from dataclasses import dataclass -from piccolo.columns.base import Selectable from piccolo.columns.column_types import ( JSON, JSONB, @@ -12,6 +11,7 @@ ForeignKey, LazyTableReference, ) +from piccolo.querystring import QueryString, Selectable from piccolo.utils.list import flatten from piccolo.utils.sync import run_sync @@ -56,7 +56,9 @@ def __init__( for column in columns ) - def get_select_string(self, engine_type: str, with_alias=True) -> str: + def get_select_string( + self, engine_type: str, with_alias=True + ) -> QueryString: m2m_table_name_with_schema = ( self.m2m._meta.resolved_joining_table._meta.get_formatted_tablename() # noqa: E501 ) # noqa: E501 @@ -90,28 +92,33 @@ def get_select_string(self, engine_type: str, with_alias=True) -> str: if engine_type in ("postgres", "cockroach"): if self.as_list: column_name = self.columns[0]._meta.db_column_name - return f""" + return QueryString( + f""" ARRAY( SELECT "inner_{table_2_name}"."{column_name}" FROM {inner_select} ) AS "{m2m_relationship_name}" """ + ) elif not self.serialisation_safe: column_name = table_2_pk_name - return f""" + return QueryString( + f""" ARRAY( SELECT "inner_{table_2_name}"."{column_name}" FROM {inner_select} ) AS "{m2m_relationship_name}" """ + ) else: column_names = ", ".join( f'"inner_{table_2_name}"."{column._meta.db_column_name}"' for column in self.columns ) - return f""" + return QueryString( + f""" ( SELECT JSON_AGG({m2m_relationship_name}_results) FROM ( @@ -119,13 +126,15 @@ def get_select_string(self, engine_type: str, with_alias=True) -> str: ) AS "{m2m_relationship_name}_results" ) AS "{m2m_relationship_name}" """ + ) elif engine_type == "sqlite": if len(self.columns) > 1 or not self.serialisation_safe: column_name = table_2_pk_name else: column_name = self.columns[0]._meta.db_column_name - return f""" + return QueryString( + f""" ( SELECT group_concat( "inner_{table_2_name}"."{column_name}" @@ -134,6 +143,7 @@ def get_select_string(self, engine_type: str, with_alias=True) -> str: ) AS "{m2m_relationship_name} [M2M]" """ + ) else: raise ValueError(f"{engine_type} is an unrecognised engine type") diff --git a/piccolo/columns/readable.py b/piccolo/columns/readable.py index 2748648d8..ebd32bf51 100644 --- a/piccolo/columns/readable.py +++ b/piccolo/columns/readable.py @@ -3,7 +3,7 @@ import typing as t from dataclasses import dataclass -from piccolo.columns.base import Selectable +from piccolo.querystring import QueryString, Selectable if t.TYPE_CHECKING: # pragma: no cover from piccolo.columns.base import Column @@ -27,25 +27,27 @@ def _columns_string(self) -> str: i._meta.get_full_name(with_alias=False) for i in self.columns ) - def _get_string(self, operator: str) -> str: - return ( + def _get_string(self, operator: str) -> QueryString: + return QueryString( f"{operator}('{self.template}', {self._columns_string}) AS " f"{self.output_name}" ) @property - def sqlite_string(self) -> str: + def sqlite_string(self) -> QueryString: return self._get_string(operator="PRINTF") @property - def postgres_string(self) -> str: + def postgres_string(self) -> QueryString: return self._get_string(operator="FORMAT") @property - def cockroach_string(self) -> str: + def cockroach_string(self) -> QueryString: return self._get_string(operator="FORMAT") - def get_select_string(self, engine_type: str, with_alias=True) -> str: + def get_select_string( + self, engine_type: str, with_alias=True + ) -> QueryString: try: return getattr(self, f"{engine_type}_string") except AttributeError as e: diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 31c38fc6a..f11b16033 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -63,8 +63,8 @@ def __init__(self, sql: str, *args: t.Any) -> None: def get_select_string( self, engine_type: str, with_alias: bool = True - ) -> str: - return self.querystring.__str__() + ) -> QueryString: + return self.querystring class Avg(Selectable): @@ -96,9 +96,9 @@ def __init__(self, column: Column, alias: str = "avg"): def get_select_string( self, engine_type: str, with_alias: bool = True - ) -> str: + ) -> QueryString: column_name = self.column._meta.get_full_name(with_alias=False) - return f'AVG({column_name}) AS "{self._alias}"' + return QueryString(f'AVG({column_name}) AS "{self._alias}"') class Count(Selectable): @@ -156,7 +156,7 @@ def __init__( def get_select_string( self, engine_type: str, with_alias: bool = True - ) -> str: + ) -> QueryString: expression: str if self.distinct: @@ -180,7 +180,7 @@ def get_select_string( else: expression = "*" - return f'COUNT({expression}) AS "{self._alias}"' + return QueryString(f'COUNT({expression}) AS "{self._alias}"') class Max(Selectable): @@ -211,9 +211,9 @@ def __init__(self, column: Column, alias: str = "max"): def get_select_string( self, engine_type: str, with_alias: bool = True - ) -> str: + ) -> QueryString: column_name = self.column._meta.get_full_name(with_alias=False) - return f'MAX({column_name}) AS "{self._alias}"' + return QueryString(f'MAX({column_name}) AS "{self._alias}"') class Min(Selectable): @@ -242,9 +242,9 @@ def __init__(self, column: Column, alias: str = "min"): def get_select_string( self, engine_type: str, with_alias: bool = True - ) -> str: + ) -> QueryString: column_name = self.column._meta.get_full_name(with_alias=False) - return f'MIN({column_name}) AS "{self._alias}"' + return QueryString(f'MIN({column_name}) AS "{self._alias}"') class Sum(Selectable): @@ -278,9 +278,9 @@ def __init__(self, column: Column, alias: str = "sum"): def get_select_string( self, engine_type: str, with_alias: bool = True - ) -> str: + ) -> QueryString: column_name = self.column._meta.get_full_name(with_alias=False) - return f'SUM({column_name}) AS "{self._alias}"' + return QueryString(f'SUM({column_name}) AS "{self._alias}"') OptionalDict = t.Optional[t.Dict[str, t.Any]] @@ -765,11 +765,10 @@ def default_querystrings(self) -> t.Sequence[QueryString]: engine_type = self.table._meta.db.engine_type - select_strings: t.List[str] = [ + select_strings: t.List[QueryString] = [ c.get_select_string(engine_type=engine_type) for c in self.columns_delegate.selected_columns ] - columns_str = ", ".join(select_strings) ####################################################################### @@ -783,7 +782,9 @@ def default_querystrings(self) -> t.Sequence[QueryString]: query += "{}" args.append(distinct.querystring) + columns_str = ", ".join("{}" for i in select_strings) query += f" {columns_str} FROM {self.table._meta.get_formatted_tablename()}" # noqa: E501 + args.extend(select_strings) for join in joins: query += f" {join}" diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index 8d7c6a4a9..b64afb206 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -9,13 +9,14 @@ from piccolo.columns import And, Column, Or, Where from piccolo.columns.column_types import ForeignKey +from piccolo.columns.combination import WhereRaw from piccolo.custom_types import Combinable from piccolo.querystring import QueryString from piccolo.utils.list import flatten from piccolo.utils.sql_values import convert_to_sql_value if t.TYPE_CHECKING: # pragma: no cover - from piccolo.columns.base import Selectable + from piccolo.querystring import Selectable from piccolo.table import Table # noqa @@ -265,6 +266,10 @@ def where(self, *where: Combinable): "`.where(MyTable.some_column.is_null())`." ) + if isinstance(arg, QueryString): + # If a raw QueryString is passed in. + arg = WhereRaw(arg.template, *arg.args) + self._where = And(self._where, arg) if self._where else arg diff --git a/piccolo/querystring.py b/piccolo/querystring.py index 3c23d86dc..9a5f63a4d 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -1,6 +1,7 @@ from __future__ import annotations import typing as t +from abc import ABCMeta, abstractmethod from dataclasses import dataclass from datetime import datetime from importlib.util import find_spec @@ -17,6 +18,32 @@ apgUUID = UUID +class Selectable(metaclass=ABCMeta): + """ + Anything which inherits from this can be used in a select query. + """ + + _alias: t.Optional[str] + + @abstractmethod + def get_select_string( + self, engine_type: str, with_alias: bool = True + ) -> QueryString: + """ + In a query, what to output after the select statement - could be a + column name, a sub query, a function etc. For a column it will be the + column name. + """ + raise NotImplementedError() + + def as_alias(self, alias: str) -> Selectable: + """ + Allows column names to be changed in the result of a select. + """ + self._alias = alias + return self + + @dataclass class Unquoted: """ @@ -42,7 +69,7 @@ class Fragment: no_arg: bool = False -class QueryString: +class QueryString(Selectable): """ When we're composing complex queries, we're combining QueryStrings, rather than concatenating strings directly. The reason for this is QueryStrings @@ -143,7 +170,7 @@ def bundle( fragment.no_arg = True bundled.append(fragment) else: - if isinstance(value, self.__class__): + if isinstance(value, QueryString): fragment.no_arg = True bundled.append(fragment) @@ -195,3 +222,22 @@ def freeze(self, engine_type: str = "postgres"): self._frozen_compiled_strings = self.compile_string( engine_type=engine_type ) + + ########################################################################### + + def get_select_string( + self, engine_type: str, with_alias: bool = True + ) -> QueryString: + return self + + ########################################################################### + # Basic logic + + def __eq__(self, value) -> QueryString: + return QueryString("{} = {}", self, value) + + def __ne__(self, value) -> QueryString: + return QueryString("{} != {}", self, value) + + def is_in(self, value) -> QueryString: + return QueryString("{} != {}", self, value) diff --git a/piccolo/table.py b/piccolo/table.py index b4fcbf942..6c0984dff 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -56,7 +56,7 @@ from piccolo.utils.warnings import colored_warning if t.TYPE_CHECKING: # pragma: no cover - from piccolo.columns import Selectable + from piccolo.querystring import Selectable PROTECTED_TABLENAMES = ("user",) TABLENAME_WARNING = ( From 45c68e547b53aa1b1cb48c857257196ea1c2ad8f Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 28 May 2024 17:18:43 +0100 Subject: [PATCH 08/39] fix some linter issues --- piccolo/columns/column_types.py | 7 ++----- piccolo/querystring.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 3939ec21b..b78eb3f1a 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -60,7 +60,7 @@ class Band(Table): from piccolo.columns.operators.comparison import ArrayAll, ArrayAny from piccolo.columns.operators.string import Concat from piccolo.columns.reference import LazyTableReference -from piccolo.querystring import QueryString, Selectable, Unquoted +from piccolo.querystring import QueryString, Unquoted from piccolo.utils.encoding import dump_json from piccolo.utils.warnings import colored_warning @@ -287,16 +287,13 @@ def get_querystring( ############################################################################### -StringColumn = t.TypeVar("StringColumn", "Varchar", "Text") - - class Function(QueryString): function_name: str columns: t.List[Column] def __init__( self, - identifier: t.Union[StringColumn, QueryString, str], + identifier: t.Union[Column, QueryString, str], alias: t.Optional[str] = None, ): self._alias = alias diff --git a/piccolo/querystring.py b/piccolo/querystring.py index 9a5f63a4d..cb7721732 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -233,11 +233,17 @@ def get_select_string( ########################################################################### # Basic logic - def __eq__(self, value) -> QueryString: + def __eq__(self, value) -> QueryString: # type: ignore[override] return QueryString("{} = {}", self, value) - def __ne__(self, value) -> QueryString: + def __ne__(self, value) -> QueryString: # type: ignore[override] return QueryString("{} != {}", self, value) + def __add__(self, value) -> QueryString: + return QueryString("{} + {}", self, value) + + def __sub__(self, value) -> QueryString: + return QueryString("{} - {}", self, value) + def is_in(self, value) -> QueryString: return QueryString("{} != {}", self, value) From 0f4c8f12f322a66efda4bc7de9b503551b78f8f4 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 28 May 2024 21:21:20 +0100 Subject: [PATCH 09/39] fix linter errors --- piccolo/query/base.py | 4 ++-- piccolo/querystring.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/piccolo/query/base.py b/piccolo/query/base.py index b10d42ee0..3e1e93a8d 100644 --- a/piccolo/query/base.py +++ b/piccolo/query/base.py @@ -91,8 +91,8 @@ async def _process_results(self, results) -> QueryResponseType: json_column_names.append(column._meta.name) elif len(column._meta.call_chain) > 0: json_column_names.append( - column.get_select_string( - engine_type=column._meta.engine_type + column._meta.get_full_name( + with_alias=False, include_quotes=False ) ) else: diff --git a/piccolo/querystring.py b/piccolo/querystring.py index cb7721732..d95dbc3c3 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -23,6 +23,8 @@ class Selectable(metaclass=ABCMeta): Anything which inherits from this can be used in a select query. """ + __slots__ = ("_alias",) + _alias: t.Optional[str] @abstractmethod From 4eab44877cde2c564ee0658e45abc7b0d85f3b17 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 28 May 2024 21:22:11 +0100 Subject: [PATCH 10/39] add album table --- piccolo/apps/playground/commands/run.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/piccolo/apps/playground/commands/run.py b/piccolo/apps/playground/commands/run.py index b4bc23f70..ded5f408e 100644 --- a/piccolo/apps/playground/commands/run.py +++ b/piccolo/apps/playground/commands/run.py @@ -128,7 +128,30 @@ def get_readable(cls) -> Readable: ) -TABLES = (Manager, Band, Venue, Concert, Ticket, DiscountCode, RecordingStudio) +class Album(Table): + name = Varchar() + band = ForeignKey(Band) + release_date = Timestamp() + recorded_at = ForeignKey(RecordingStudio) + + @classmethod + def get_readable(cls) -> Readable: + return Readable( + template="%s - %s", + columns=[cls.name, cls.band._.name], + ) + + +TABLES = ( + Manager, + Band, + Venue, + Concert, + Ticket, + DiscountCode, + RecordingStudio, + Album, +) def populate(): From 7a17c2a26d23db2271fd709116eb337550115545 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Tue, 28 May 2024 22:30:18 +0100 Subject: [PATCH 11/39] fix `load_json` on joined tables --- piccolo/query/base.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/piccolo/query/base.py b/piccolo/query/base.py index 3e1e93a8d..c169fde0e 100644 --- a/piccolo/query/base.py +++ b/piccolo/query/base.py @@ -87,13 +87,9 @@ async def _process_results(self, results) -> QueryResponseType: for column in json_columns: if column._alias is not None: json_column_names.append(column._alias) - elif column.json_operator is not None: - json_column_names.append(column._meta.name) elif len(column._meta.call_chain) > 0: json_column_names.append( - column._meta.get_full_name( - with_alias=False, include_quotes=False - ) + column._meta.get_default_alias().replace("$", ".") ) else: json_column_names.append(column._meta.name) From bfa6c5c5e2641e66b4ed76af41071a2906483aa5 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 May 2024 08:21:46 +0100 Subject: [PATCH 12/39] move logic for `get_select_string` from `Function` to `QueryString` --- piccolo/columns/column_types.py | 8 -------- piccolo/querystring.py | 5 ++++- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index b78eb3f1a..7be4fcd7d 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -317,14 +317,6 @@ def __init__( else: raise ValueError("Unrecognised function type") - def get_select_string( - self, engine_type: str, with_alias: bool = True - ) -> QueryString: - if with_alias and self._alias: - return QueryString("{} AS " + self._alias, self) - else: - return self - class Upper(Function): function_name = "UPPER" diff --git a/piccolo/querystring.py b/piccolo/querystring.py index d95dbc3c3..0efb5835b 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -230,7 +230,10 @@ def freeze(self, engine_type: str = "postgres"): def get_select_string( self, engine_type: str, with_alias: bool = True ) -> QueryString: - return self + if with_alias and self._alias: + return QueryString("{} AS " + self._alias, self) + else: + return self ########################################################################### # Basic logic From 455b9b3b76a801c1215f2aaf24e575a896dbb081 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 May 2024 11:04:01 +0100 Subject: [PATCH 13/39] use columns in querystring for joins --- piccolo/query/methods/select.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index f11b16033..995611e46 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -682,6 +682,13 @@ def _get_joins(self, columns: t.Sequence[Selectable]) -> t.List[str]: for readable in readables: columns += readable.columns + querystrings: t.List[QueryString] = [ + i for i in columns if isinstance(i, QueryString) + ] + for querystring in querystrings: + if querystring_columns := getattr(querystring, "columns"): + columns += querystring_columns + for column in columns: if not isinstance(column, Column): continue From 4cbc5450f107caa29a317338987c3e150bca5db8 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 May 2024 11:07:45 +0100 Subject: [PATCH 14/39] add `not_in` to `querystring` --- piccolo/querystring.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/piccolo/querystring.py b/piccolo/querystring.py index 0efb5835b..3e5847d6b 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -251,4 +251,7 @@ def __sub__(self, value) -> QueryString: return QueryString("{} - {}", self, value) def is_in(self, value) -> QueryString: - return QueryString("{} != {}", self, value) + return QueryString("{} IN {}", self, value) + + def not_in(self, value) -> QueryString: + return QueryString("{} NOT IN {}", self, value) From 347d2fd1976562810647fea43d0781de9c2688c8 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 May 2024 11:07:53 +0100 Subject: [PATCH 15/39] add `get_where_string` --- piccolo/querystring.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/piccolo/querystring.py b/piccolo/querystring.py index 3e5847d6b..2a21ef856 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -235,6 +235,11 @@ def get_select_string( else: return self + def get_where_string(self, engine_type: str) -> QueryString: + return self.get_select_string( + engine_type=engine_type, with_alias=False + ) + ########################################################################### # Basic logic From 27079367ac138c5810b41517758b9efc3bf50e00 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 May 2024 11:08:02 +0100 Subject: [PATCH 16/39] set `_alias` in querystring __init__ --- piccolo/querystring.py | 1 + 1 file changed, 1 insertion(+) diff --git a/piccolo/querystring.py b/piccolo/querystring.py index 2a21ef856..ed191c385 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -118,6 +118,7 @@ def __init__( self._frozen_compiled_strings: t.Optional[ t.Tuple[str, t.List[t.Any]] ] = None + self._alias = None def __str__(self): """ From d76f84b8e30c47ed6bb247ca6ff673412700976c Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 May 2024 11:18:24 +0100 Subject: [PATCH 17/39] refactor `table_alias` --- piccolo/columns/base.py | 10 ++++++++-- piccolo/query/methods/select.py | 9 ++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/piccolo/columns/base.py b/piccolo/columns/base.py index 50ff9b1a0..ce452260c 100644 --- a/piccolo/columns/base.py +++ b/piccolo/columns/base.py @@ -205,7 +205,6 @@ def table(self) -> t.Type[Table]: # Used by Foreign Keys: call_chain: t.List["ForeignKey"] = field(default_factory=list) - table_alias: t.Optional[str] = None ########################################################################### @@ -260,7 +259,7 @@ def _get_path(self, include_quotes: bool = False): column_name = self.db_column_name if self.call_chain: - table_alias = self.call_chain[-1]._meta.table_alias + table_alias = self.call_chain[-1].table_alias if include_quotes: return f'"{table_alias}"."{column_name}"' else: @@ -884,6 +883,13 @@ def get_sql_value(self, value: t.Any) -> t.Any: def column_type(self): return self.__class__.__name__.upper() + @property + def table_alias(self) -> str: + return "$".join( + f"{_key._meta.table._meta.tablename}${_key._meta.name}" + for _key in [*self._meta.call_chain, self] + ) + @property def ddl(self) -> str: """ diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 995611e46..04187563c 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -695,17 +695,12 @@ def _get_joins(self, columns: t.Sequence[Selectable]) -> t.List[str]: _joins: t.List[str] = [] for index, key in enumerate(column._meta.call_chain, 0): - table_alias = "$".join( - f"{_key._meta.table._meta.tablename}${_key._meta.name}" - for _key in column._meta.call_chain[: index + 1] - ) - - key._meta.table_alias = table_alias + table_alias = key.table_alias if index > 0: left_tablename = column._meta.call_chain[ index - 1 - ]._meta.table_alias + ].table_alias else: left_tablename = ( key._meta.table._meta.get_formatted_tablename() From 6dc497a096f3cecb554ae2cc04820ad99c938e18 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 May 2024 11:30:36 +0100 Subject: [PATCH 18/39] move functions into a new folder --- piccolo/columns/column_types.py | 39 ----------------------------- piccolo/query/functions/__init__.py | 0 piccolo/query/functions/base.py | 35 ++++++++++++++++++++++++++ piccolo/query/functions/string.py | 9 +++++++ 4 files changed, 44 insertions(+), 39 deletions(-) create mode 100644 piccolo/query/functions/__init__.py create mode 100644 piccolo/query/functions/base.py create mode 100644 piccolo/query/functions/string.py diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 7be4fcd7d..96a921a75 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -287,45 +287,6 @@ def get_querystring( ############################################################################### -class Function(QueryString): - function_name: str - columns: t.List[Column] - - def __init__( - self, - identifier: t.Union[Column, QueryString, str], - alias: t.Optional[str] = None, - ): - self._alias = alias - - if isinstance(identifier, Column): - if not alias: - self._alias = identifier._meta.get_default_alias() - - # We track any columns just in case we need to perform joins - self.columns = [identifier, *getattr(self, "columns", [])] - - column_full_name = identifier._meta.get_full_name(with_alias=False) - super().__init__(f"{self.function_name}({column_full_name})") - elif isinstance(identifier, QueryString): - if identifier._alias: - self._alias = identifier._alias - - super().__init__(f"{self.function_name}({{}})", identifier) - elif isinstance(identifier, str): - super().__init__(f"{self.function_name}({{}})", identifier) - else: - raise ValueError("Unrecognised function type") - - -class Upper(Function): - function_name = "UPPER" - - -class Lower(Function): - function_name = "LOWER" - - class Varchar(Column): """ Used for storing text when you want to enforce character length limits. diff --git a/piccolo/query/functions/__init__.py b/piccolo/query/functions/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/piccolo/query/functions/base.py b/piccolo/query/functions/base.py new file mode 100644 index 000000000..984fc90df --- /dev/null +++ b/piccolo/query/functions/base.py @@ -0,0 +1,35 @@ +import typing as t + +from piccolo.columns.base import Column +from piccolo.querystring import QueryString + + +class Function(QueryString): + function_name: str + columns: t.List[Column] + + def __init__( + self, + identifier: t.Union[Column, QueryString, str], + alias: t.Optional[str] = None, + ): + self._alias = alias + + if isinstance(identifier, Column): + if not alias: + self._alias = identifier._meta.get_default_alias() + + # We track any columns just in case we need to perform joins + self.columns = [identifier, *getattr(self, "columns", [])] + + column_full_name = identifier._meta.get_full_name(with_alias=False) + super().__init__(f"{self.function_name}({column_full_name})") + elif isinstance(identifier, QueryString): + if identifier._alias: + self._alias = identifier._alias + + super().__init__(f"{self.function_name}({{}})", identifier) + elif isinstance(identifier, str): + super().__init__(f"{self.function_name}({{}})", identifier) + else: + raise ValueError("Unrecognised function type") diff --git a/piccolo/query/functions/string.py b/piccolo/query/functions/string.py new file mode 100644 index 000000000..4149662af --- /dev/null +++ b/piccolo/query/functions/string.py @@ -0,0 +1,9 @@ +from .base import Function + + +class Upper(Function): + function_name = "UPPER" + + +class Lower(Function): + function_name = "LOWER" From ef8e61b3441b8fa4392cf3e013a15027decfdc72 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 May 2024 11:31:40 +0100 Subject: [PATCH 19/39] re-export `Upper` and `Lower` --- piccolo/query/functions/__init__.py | 3 +++ piccolo/query/functions/string.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/piccolo/query/functions/__init__.py b/piccolo/query/functions/__init__.py index e69de29bb..9d111108c 100644 --- a/piccolo/query/functions/__init__.py +++ b/piccolo/query/functions/__init__.py @@ -0,0 +1,3 @@ +from .string import Lower, Upper + +__all__ = ("Upper", "Lower") diff --git a/piccolo/query/functions/string.py b/piccolo/query/functions/string.py index 4149662af..139832645 100644 --- a/piccolo/query/functions/string.py +++ b/piccolo/query/functions/string.py @@ -7,3 +7,6 @@ class Upper(Function): class Lower(Function): function_name = "LOWER" + + +__all__ = ("Upper", "Lower") From 95b8c7db4b4b34593f885650de6d975149763673 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 May 2024 11:38:02 +0100 Subject: [PATCH 20/39] add ltrim and rtrim functions --- piccolo/query/functions/string.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/piccolo/query/functions/string.py b/piccolo/query/functions/string.py index 139832645..600ceba72 100644 --- a/piccolo/query/functions/string.py +++ b/piccolo/query/functions/string.py @@ -9,4 +9,12 @@ class Lower(Function): function_name = "LOWER" -__all__ = ("Upper", "Lower") +class Ltrim(Function): + function_name = "LTRIM" + + +class Rtrim(Function): + function_name = "RTRIM" + + +__all__ = ("Upper", "Lower", "Ltrim", "Rtrim") From ec5bab920e1de1e8f4e55d7a88b0283755062206 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 May 2024 11:53:41 +0100 Subject: [PATCH 21/39] add more functions --- piccolo/query/functions/__init__.py | 10 ++++-- piccolo/query/functions/string.py | 47 +++++++++++++++++++++++++++-- 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/piccolo/query/functions/__init__.py b/piccolo/query/functions/__init__.py index 9d111108c..75a31bec0 100644 --- a/piccolo/query/functions/__init__.py +++ b/piccolo/query/functions/__init__.py @@ -1,3 +1,9 @@ -from .string import Lower, Upper +from .string import Length, Lower, Ltrim, Rtrim, Upper -__all__ = ("Upper", "Lower") +__all__ = ( + "Length", + "Lower", + "Ltrim", + "Rtrim", + "Upper", +) diff --git a/piccolo/query/functions/string.py b/piccolo/query/functions/string.py index 600ceba72..117557a1d 100644 --- a/piccolo/query/functions/string.py +++ b/piccolo/query/functions/string.py @@ -1,20 +1,61 @@ +""" +These functions mirror their counterparts in the Postgresql docs: + +https://www.postgresql.org/docs/current/functions-string.html + +""" + from .base import Function -class Upper(Function): - function_name = "UPPER" +class Length(Function): + """ + Returns the number of characters in the string. + """ + + function_name = "LENGTH" class Lower(Function): + """ + Converts the string to all lower case, according to the rules of the + database's locale. + """ + function_name = "LOWER" class Ltrim(Function): + """ + Removes the longest string containing only characters in characters (a + space by default) from the start of string. + """ + function_name = "LTRIM" class Rtrim(Function): + """ + Removes the longest string containing only characters in characters (a + space by default) from the end of string. + """ + function_name = "RTRIM" -__all__ = ("Upper", "Lower", "Ltrim", "Rtrim") +class Upper(Function): + """ + Converts the string to all upper case, according to the rules of the + database's locale. + """ + + function_name = "UPPER" + + +__all__ = ( + "Length", + "Lower", + "Ltrim", + "Rtrim", + "Upper", +) From 53edbbb310e64e0554eaddd05c8109addb489474 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 May 2024 12:15:08 +0100 Subject: [PATCH 22/39] improve error message --- piccolo/query/functions/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piccolo/query/functions/base.py b/piccolo/query/functions/base.py index 984fc90df..36a17e8ae 100644 --- a/piccolo/query/functions/base.py +++ b/piccolo/query/functions/base.py @@ -32,4 +32,4 @@ def __init__( elif isinstance(identifier, str): super().__init__(f"{self.function_name}({{}})", identifier) else: - raise ValueError("Unrecognised function type") + raise ValueError("Unrecognised identifier type") From 5b5cab08b8345ffb389daeede8834b50e06b118f Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 May 2024 12:15:26 +0100 Subject: [PATCH 23/39] add default value for `getattr` when fetching querystring columns --- piccolo/query/methods/select.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 04187563c..124b3ca6c 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -686,7 +686,7 @@ def _get_joins(self, columns: t.Sequence[Selectable]) -> t.List[str]: i for i in columns if isinstance(i, QueryString) ] for querystring in querystrings: - if querystring_columns := getattr(querystring, "columns"): + if querystring_columns := getattr(querystring, "columns", []): columns += querystring_columns for column in columns: From c73e215044cf51f4d3d70f4df67c2ac1dfb28d18 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 May 2024 12:15:31 +0100 Subject: [PATCH 24/39] add initial tests --- tests/query/functions/test_string.py | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 tests/query/functions/test_string.py diff --git a/tests/query/functions/test_string.py b/tests/query/functions/test_string.py new file mode 100644 index 000000000..c26e7701c --- /dev/null +++ b/tests/query/functions/test_string.py @@ -0,0 +1,36 @@ +from unittest import TestCase + +from piccolo.query.functions.string import Upper +from piccolo.table import create_db_tables_sync, drop_db_tables_sync +from tests.example_apps.music.tables import Band, Manager + + +class TestUpperFunction(TestCase): + + tables = (Band, Manager) + + def setUp(self) -> None: + create_db_tables_sync(*self.tables) + + manager = Manager({Manager.name: "Guido"}) + manager.save().run_sync() + + band = Band({Band.name: "Pythonistas", Band.manager: manager}) + band.save().run_sync() + + def tearDown(self) -> None: + drop_db_tables_sync(*self.tables) + + def test_column(self): + """ + Make sure we can uppercase a column's value. + """ + response = Band.select(Upper(Band.name)).run_sync() + self.assertEqual(response, [{"upper": "PYTHONISTAS"}]) + + def test_joined_column(self): + """ + Make sure we can uppercase a column's value from a joined table. + """ + response = Band.select(Upper(Band.manager._.name)).run_sync() + self.assertEqual(response, [{"upper": "GUIDO"}]) From 1b0c0d5feafffcb291477ddeb9a20720401a4de2 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 May 2024 19:00:58 +0100 Subject: [PATCH 25/39] add a test for alias --- tests/query/functions/test_string.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/query/functions/test_string.py b/tests/query/functions/test_string.py index c26e7701c..70408a94f 100644 --- a/tests/query/functions/test_string.py +++ b/tests/query/functions/test_string.py @@ -28,6 +28,10 @@ def test_column(self): response = Band.select(Upper(Band.name)).run_sync() self.assertEqual(response, [{"upper": "PYTHONISTAS"}]) + def test_alias(self): + response = Band.select(Upper(Band.name, alias="name")).run_sync() + self.assertEqual(response, [{"name": "PYTHONISTAS"}]) + def test_joined_column(self): """ Make sure we can uppercase a column's value from a joined table. From 3000aedbc93e907b74fd64d95e7feef4fa4b2b9a Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 May 2024 19:02:27 +0100 Subject: [PATCH 26/39] deprecate `Unquoted` - `QueryString` can be used directly --- piccolo/columns/column_types.py | 8 ++++---- piccolo/querystring.py | 26 ++++++++------------------ piccolo/table.py | 30 +++++++----------------------- 3 files changed, 19 insertions(+), 45 deletions(-) diff --git a/piccolo/columns/column_types.py b/piccolo/columns/column_types.py index 96a921a75..add0c6f5c 100644 --- a/piccolo/columns/column_types.py +++ b/piccolo/columns/column_types.py @@ -60,7 +60,7 @@ class Band(Table): from piccolo.columns.operators.comparison import ArrayAll, ArrayAny from piccolo.columns.operators.string import Concat from piccolo.columns.reference import LazyTableReference -from piccolo.querystring import QueryString, Unquoted +from piccolo.querystring import QueryString from piccolo.utils.encoding import dump_json from piccolo.utils.warnings import colored_warning @@ -752,8 +752,8 @@ def __set__(self, obj, value: t.Union[int, None]): ############################################################################### -DEFAULT = Unquoted("DEFAULT") -NULL = Unquoted("null") +DEFAULT = QueryString("DEFAULT") +NULL = QueryString("null") class Serial(Column): @@ -778,7 +778,7 @@ def default(self): if engine_type == "postgres": return DEFAULT elif engine_type == "cockroach": - return Unquoted("unique_rowid()") + return QueryString("unique_rowid()") elif engine_type == "sqlite": return NULL raise Exception("Unrecognized engine type") diff --git a/piccolo/querystring.py b/piccolo/querystring.py index ed191c385..41db206c0 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -46,24 +46,6 @@ def as_alias(self, alias: str) -> Selectable: return self -@dataclass -class Unquoted: - """ - Used when we want the value to be unquoted because it's a Postgres - keyword - for example DEFAULT. - """ - - __slots__ = ("value",) - - value: str - - def __repr__(self): - return f"{self.value}" - - def __str__(self): - return f"{self.value}" - - @dataclass class Fragment: prefix: str @@ -261,3 +243,11 @@ def is_in(self, value) -> QueryString: def not_in(self, value) -> QueryString: return QueryString("{} NOT IN {}", self, value) + + +class Unquoted(QueryString): + """ + This is deprecated - just use QueryString directly. + """ + + pass diff --git a/piccolo/table.py b/piccolo/table.py index 6c0984dff..7882db95e 100644 --- a/piccolo/table.py +++ b/piccolo/table.py @@ -48,7 +48,7 @@ from piccolo.query.methods.indexes import Indexes from piccolo.query.methods.objects import First from piccolo.query.methods.refresh import Refresh -from piccolo.querystring import QueryString, Unquoted +from piccolo.querystring import QueryString from piccolo.utils import _camel_to_snake from piccolo.utils.graphlib import TopologicalSorter from piccolo.utils.sql_values import convert_to_sql_value @@ -796,30 +796,14 @@ def querystring(self) -> QueryString: """ Used when inserting rows. """ - args_dict = {} - for col in self._meta.columns: - column_name = col._meta.name - value = convert_to_sql_value(value=self[column_name], column=col) - args_dict[column_name] = value - - def is_unquoted(arg): - return isinstance(arg, Unquoted) - - # Strip out any args which are unquoted. - filtered_args = [i for i in args_dict.values() if not is_unquoted(i)] + args = [ + convert_to_sql_value(value=self[column._meta.name], column=column) + for column in self._meta.columns + ] # If unquoted, dump it straight into the query. - query = ",".join( - [ - ( - args_dict[column._meta.name].value - if is_unquoted(args_dict[column._meta.name]) - else "{}" - ) - for column in self._meta.columns - ] - ) - return QueryString(f"({query})", *filtered_args) + query = ",".join(["{}" for _ in args]) + return QueryString(f"({query})", *args) def __str__(self) -> str: return self.querystring.__str__() From 48fabb0780e8579958ae014eda85fa1d0d824fb5 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 May 2024 19:02:49 +0100 Subject: [PATCH 27/39] simplify alias handling for `Function` --- piccolo/query/functions/base.py | 22 ++++++++++++++-------- piccolo/querystring.py | 7 ++++++- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/piccolo/query/functions/base.py b/piccolo/query/functions/base.py index 36a17e8ae..753226158 100644 --- a/piccolo/query/functions/base.py +++ b/piccolo/query/functions/base.py @@ -13,23 +13,29 @@ def __init__( identifier: t.Union[Column, QueryString, str], alias: t.Optional[str] = None, ): - self._alias = alias - if isinstance(identifier, Column): - if not alias: - self._alias = identifier._meta.get_default_alias() - # We track any columns just in case we need to perform joins self.columns = [identifier, *getattr(self, "columns", [])] column_full_name = identifier._meta.get_full_name(with_alias=False) - super().__init__(f"{self.function_name}({column_full_name})") + super().__init__( + f"{self.function_name}({column_full_name})", + alias=alias, + ) elif isinstance(identifier, QueryString): if identifier._alias: self._alias = identifier._alias - super().__init__(f"{self.function_name}({{}})", identifier) + super().__init__( + f"{self.function_name}({{}})", + identifier, + alias=alias, + ) elif isinstance(identifier, str): - super().__init__(f"{self.function_name}({{}})", identifier) + super().__init__( + f"{self.function_name}({{}})", + identifier, + alias=alias, + ) else: raise ValueError("Unrecognised identifier type") diff --git a/piccolo/querystring.py b/piccolo/querystring.py index 41db206c0..c766fde30 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -75,6 +75,7 @@ def __init__( *args: t.Any, query_type: str = "generic", table: t.Optional[t.Type[Table]] = None, + alias: t.Optional[str] = None, ) -> None: """ :param template: @@ -100,7 +101,11 @@ def __init__( self._frozen_compiled_strings: t.Optional[ t.Tuple[str, t.List[t.Any]] ] = None - self._alias = None + self._alias = alias + + def as_alias(self, alias: str) -> QueryString: + self._alias = alias + return self def __str__(self): """ From 67415ebeab21f05b20f460341e8f7e36af70b400 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 May 2024 19:06:05 +0100 Subject: [PATCH 28/39] don't get alias from child `QueryString` --- piccolo/query/functions/base.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/piccolo/query/functions/base.py b/piccolo/query/functions/base.py index 753226158..91c97dd0e 100644 --- a/piccolo/query/functions/base.py +++ b/piccolo/query/functions/base.py @@ -22,16 +22,7 @@ def __init__( f"{self.function_name}({column_full_name})", alias=alias, ) - elif isinstance(identifier, QueryString): - if identifier._alias: - self._alias = identifier._alias - - super().__init__( - f"{self.function_name}({{}})", - identifier, - alias=alias, - ) - elif isinstance(identifier, str): + elif isinstance(identifier, (QueryString, str)): super().__init__( f"{self.function_name}({{}})", identifier, From 6bd4aab094fcc260ef73d9956374d6ea047d39d1 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 May 2024 19:15:02 +0100 Subject: [PATCH 29/39] add `Reverse` function --- piccolo/query/functions/string.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/piccolo/query/functions/string.py b/piccolo/query/functions/string.py index 117557a1d..e5102cabd 100644 --- a/piccolo/query/functions/string.py +++ b/piccolo/query/functions/string.py @@ -34,6 +34,14 @@ class Ltrim(Function): function_name = "LTRIM" +class Reverse(Function): + """ + Return reversed string. + """ + + function_name = "REVERSE" + + class Rtrim(Function): """ Removes the longest string containing only characters in characters (a From 4a78db88b4061d95f38148d10f1dceda2263c903 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 May 2024 19:15:10 +0100 Subject: [PATCH 30/39] add `TestNested` --- tests/query/functions/test_string.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/query/functions/test_string.py b/tests/query/functions/test_string.py index 70408a94f..5b011816f 100644 --- a/tests/query/functions/test_string.py +++ b/tests/query/functions/test_string.py @@ -1,12 +1,11 @@ from unittest import TestCase -from piccolo.query.functions.string import Upper +from piccolo.query.functions.string import Reverse, Upper from piccolo.table import create_db_tables_sync, drop_db_tables_sync from tests.example_apps.music.tables import Band, Manager -class TestUpperFunction(TestCase): - +class FunctionTest(TestCase): tables = (Band, Manager) def setUp(self) -> None: @@ -21,20 +20,33 @@ def setUp(self) -> None: def tearDown(self) -> None: drop_db_tables_sync(*self.tables) + +class TestUpperFunction(FunctionTest): + def test_column(self): """ Make sure we can uppercase a column's value. """ response = Band.select(Upper(Band.name)).run_sync() - self.assertEqual(response, [{"upper": "PYTHONISTAS"}]) + self.assertListEqual(response, [{"upper": "PYTHONISTAS"}]) def test_alias(self): response = Band.select(Upper(Band.name, alias="name")).run_sync() - self.assertEqual(response, [{"name": "PYTHONISTAS"}]) + self.assertListEqual(response, [{"name": "PYTHONISTAS"}]) def test_joined_column(self): """ Make sure we can uppercase a column's value from a joined table. """ response = Band.select(Upper(Band.manager._.name)).run_sync() - self.assertEqual(response, [{"upper": "GUIDO"}]) + self.assertListEqual(response, [{"upper": "GUIDO"}]) + + +class TestNested(FunctionTest): + + def test_nested(self): + """ + Make sure we can nest functions. + """ + response = Band.select(Upper(Reverse(Band.name))).run_sync() + self.assertListEqual(response, [{"upper": "SATSINOHTYP"}]) From b566852818a8cc429b29ff3502b4df0b8e1ec2b8 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 May 2024 20:41:27 +0100 Subject: [PATCH 31/39] fix sqlite tests --- piccolo/query/functions/base.py | 2 ++ piccolo/query/functions/string.py | 3 +++ tests/query/functions/test_string.py | 5 +++++ 3 files changed, 10 insertions(+) diff --git a/piccolo/query/functions/base.py b/piccolo/query/functions/base.py index 91c97dd0e..9440554ee 100644 --- a/piccolo/query/functions/base.py +++ b/piccolo/query/functions/base.py @@ -13,6 +13,8 @@ def __init__( identifier: t.Union[Column, QueryString, str], alias: t.Optional[str] = None, ): + alias = alias or self.__class__.__name__.lower() + if isinstance(identifier, Column): # We track any columns just in case we need to perform joins self.columns = [identifier, *getattr(self, "columns", [])] diff --git a/piccolo/query/functions/string.py b/piccolo/query/functions/string.py index e5102cabd..dca3d4dfb 100644 --- a/piccolo/query/functions/string.py +++ b/piccolo/query/functions/string.py @@ -37,6 +37,9 @@ class Ltrim(Function): class Reverse(Function): """ Return reversed string. + + Not supported in SQLite. + """ function_name = "REVERSE" diff --git a/tests/query/functions/test_string.py b/tests/query/functions/test_string.py index 5b011816f..18555e3f4 100644 --- a/tests/query/functions/test_string.py +++ b/tests/query/functions/test_string.py @@ -2,6 +2,7 @@ from piccolo.query.functions.string import Reverse, Upper from piccolo.table import create_db_tables_sync, drop_db_tables_sync +from tests.base import engines_skip from tests.example_apps.music.tables import Band, Manager @@ -42,7 +43,11 @@ def test_joined_column(self): self.assertListEqual(response, [{"upper": "GUIDO"}]) +@engines_skip("sqlite") class TestNested(FunctionTest): + """ + Skip the the test for SQLite, as it doesn't support ``Reverse``. + """ def test_nested(self): """ From b103b39184935d14443a719be14f3d41714e572f Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 May 2024 20:54:22 +0100 Subject: [PATCH 32/39] improve tracking of columns within querystrings --- piccolo/query/functions/base.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/piccolo/query/functions/base.py b/piccolo/query/functions/base.py index 9440554ee..879f3364e 100644 --- a/piccolo/query/functions/base.py +++ b/piccolo/query/functions/base.py @@ -17,14 +17,23 @@ def __init__( if isinstance(identifier, Column): # We track any columns just in case we need to perform joins - self.columns = [identifier, *getattr(self, "columns", [])] + self.columns = [identifier] column_full_name = identifier._meta.get_full_name(with_alias=False) super().__init__( f"{self.function_name}({column_full_name})", alias=alias, ) - elif isinstance(identifier, (QueryString, str)): + elif isinstance(identifier, QueryString): + # Just in case the querystring passed in is also tracking columns. + self.columns = [*getattr(identifier, "columns", [])] + + super().__init__( + f"{self.function_name}({{}})", + identifier, + alias=alias, + ) + elif isinstance(identifier, str): super().__init__( f"{self.function_name}({{}})", identifier, From 7309a7b6bf2774ea488f023d4543f1a7f98f8fce Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 May 2024 21:03:40 +0100 Subject: [PATCH 33/39] increase test timeouts --- .github/workflows/tests.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index d71d78e7e..136f86680 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -13,7 +13,7 @@ on: jobs: linters: runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 60 strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] @@ -35,7 +35,7 @@ jobs: integration: runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 60 strategy: matrix: # These tests are slow, so we only run on the latest Python @@ -82,7 +82,7 @@ jobs: postgres: runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 60 strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] @@ -138,7 +138,7 @@ jobs: cockroach: runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 60 strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] @@ -172,7 +172,7 @@ jobs: sqlite: runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 60 strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] From c950f5287c433815300002ce027bb042619c8da3 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 May 2024 22:08:58 +0100 Subject: [PATCH 34/39] add missing imports --- piccolo/query/functions/__init__.py | 3 ++- piccolo/query/functions/string.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/piccolo/query/functions/__init__.py b/piccolo/query/functions/__init__.py index 75a31bec0..6912a5ff3 100644 --- a/piccolo/query/functions/__init__.py +++ b/piccolo/query/functions/__init__.py @@ -1,9 +1,10 @@ -from .string import Length, Lower, Ltrim, Rtrim, Upper +from .string import Length, Lower, Ltrim, Reverse, Rtrim, Upper __all__ = ( "Length", "Lower", "Ltrim", + "Reverse", "Rtrim", "Upper", ) diff --git a/piccolo/query/functions/string.py b/piccolo/query/functions/string.py index dca3d4dfb..556817a12 100644 --- a/piccolo/query/functions/string.py +++ b/piccolo/query/functions/string.py @@ -67,6 +67,7 @@ class Upper(Function): "Length", "Lower", "Ltrim", + "Reverse", "Rtrim", "Upper", ) From 46a6bfca5f4e471b517c70056de163faf55f8a34 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 May 2024 22:23:36 +0100 Subject: [PATCH 35/39] improve functions nested within `QueryString` --- piccolo/query/functions/base.py | 32 +++++----------------------- piccolo/querystring.py | 29 ++++++++++++++++++++++++- tests/query/functions/test_string.py | 19 +++++++++++++++++ 3 files changed, 52 insertions(+), 28 deletions(-) diff --git a/piccolo/query/functions/base.py b/piccolo/query/functions/base.py index 879f3364e..c4181aca6 100644 --- a/piccolo/query/functions/base.py +++ b/piccolo/query/functions/base.py @@ -6,7 +6,6 @@ class Function(QueryString): function_name: str - columns: t.List[Column] def __init__( self, @@ -15,29 +14,8 @@ def __init__( ): alias = alias or self.__class__.__name__.lower() - if isinstance(identifier, Column): - # We track any columns just in case we need to perform joins - self.columns = [identifier] - - column_full_name = identifier._meta.get_full_name(with_alias=False) - super().__init__( - f"{self.function_name}({column_full_name})", - alias=alias, - ) - elif isinstance(identifier, QueryString): - # Just in case the querystring passed in is also tracking columns. - self.columns = [*getattr(identifier, "columns", [])] - - super().__init__( - f"{self.function_name}({{}})", - identifier, - alias=alias, - ) - elif isinstance(identifier, str): - super().__init__( - f"{self.function_name}({{}})", - identifier, - alias=alias, - ) - else: - raise ValueError("Unrecognised identifier type") + super().__init__( + f"{self.function_name}({{}})", + identifier, + alias=alias, + ) diff --git a/piccolo/querystring.py b/piccolo/querystring.py index c766fde30..7f3f3e42a 100644 --- a/piccolo/querystring.py +++ b/piccolo/querystring.py @@ -9,6 +9,7 @@ if t.TYPE_CHECKING: # pragma: no cover from piccolo.table import Table + from piccolo.columns import Column from uuid import UUID @@ -67,6 +68,7 @@ class QueryString(Selectable): "query_type", "table", "_frozen_compiled_strings", + "columns", ) def __init__( @@ -95,13 +97,38 @@ def __init__( """ self.template = template - self.args = args self.query_type = query_type self.table = table self._frozen_compiled_strings: t.Optional[ t.Tuple[str, t.List[t.Any]] ] = None self._alias = alias + self.args, self.columns = self.process_args(args) + + def process_args( + self, args: t.Sequence[t.Any] + ) -> t.Tuple[t.Sequence[t.Any], t.Sequence[Column]]: + """ + If a Column is passed in, we convert it to the name of the column + (including joins). + """ + from piccolo.columns import Column + + processed_args = [] + columns = [] + + for arg in args: + if isinstance(arg, Column): + columns.append(arg) + arg = QueryString( + f"{arg._meta.get_full_name(with_alias=False)}" + ) + elif isinstance(arg, QueryString): + columns.extend(arg.columns) + + processed_args.append(arg) + + return (processed_args, columns) def as_alias(self, alias: str) -> QueryString: self._alias = alias diff --git a/tests/query/functions/test_string.py b/tests/query/functions/test_string.py index 18555e3f4..bae565ceb 100644 --- a/tests/query/functions/test_string.py +++ b/tests/query/functions/test_string.py @@ -1,6 +1,7 @@ from unittest import TestCase from piccolo.query.functions.string import Reverse, Upper +from piccolo.querystring import QueryString from piccolo.table import create_db_tables_sync, drop_db_tables_sync from tests.base import engines_skip from tests.example_apps.music.tables import Band, Manager @@ -55,3 +56,21 @@ def test_nested(self): """ response = Band.select(Upper(Reverse(Band.name))).run_sync() self.assertListEqual(response, [{"upper": "SATSINOHTYP"}]) + + def test_nested_with_joined_column(self): + """ + Make sure nested functions can be used on a column from a joined table. + """ + response = Band.select(Upper(Reverse(Band.manager._.name))).run_sync() + self.assertListEqual(response, [{"upper": "ODIUG"}]) + + def test_nested_within_querystring(self): + """ + If we wrap a function in a custom QueryString - make sure the columns + are still accessible, so joins are successful. + """ + response = Band.select( + QueryString("CONCAT({}, '!')", Upper(Band.manager._.name)), + ).run_sync() + + self.assertListEqual(response, [{"concat": "GUIDO!"}]) From 52e3bcb2415f9196548c849e7eddd356c3f9056a Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Wed, 29 May 2024 23:52:33 +0100 Subject: [PATCH 36/39] refactor aggregate functions to use new format --- docs/src/piccolo/query_clauses/group_by.rst | 2 +- docs/src/piccolo/query_types/count.rst | 2 +- docs/src/piccolo/query_types/select.rst | 12 +- piccolo/query/__init__.py | 5 +- piccolo/query/functions/__init__.py | 6 + piccolo/query/functions/aggregate.py | 179 ++++++++++++++++ piccolo/query/methods/__init__.py | 19 +- piccolo/query/methods/count.py | 4 +- piccolo/query/methods/select.py | 226 +------------------- tests/table/test_select.py | 11 +- 10 files changed, 218 insertions(+), 248 deletions(-) create mode 100644 piccolo/query/functions/aggregate.py diff --git a/docs/src/piccolo/query_clauses/group_by.rst b/docs/src/piccolo/query_clauses/group_by.rst index d3eb5af87..738c142b8 100644 --- a/docs/src/piccolo/query_clauses/group_by.rst +++ b/docs/src/piccolo/query_clauses/group_by.rst @@ -19,7 +19,7 @@ In the following query, we get a count of the number of bands per manager: .. code-block:: python - >>> from piccolo.query.methods.select import Count + >>> from piccolo.query.functions.aggregate import Count >>> await Band.select( ... Band.manager.name.as_alias('manager_name'), diff --git a/docs/src/piccolo/query_types/count.rst b/docs/src/piccolo/query_types/count.rst index 033667c18..794d125bc 100644 --- a/docs/src/piccolo/query_types/count.rst +++ b/docs/src/piccolo/query_types/count.rst @@ -15,7 +15,7 @@ It's equivalent to this ``select`` query: .. code-block:: python - from piccolo.query.methods.select import Count + from piccolo.query.functions.aggregate import Count >>> response = await Band.select(Count()) >>> response[0]['count'] diff --git a/docs/src/piccolo/query_types/select.rst b/docs/src/piccolo/query_types/select.rst index 1591e3580..053becd10 100644 --- a/docs/src/piccolo/query_types/select.rst +++ b/docs/src/piccolo/query_types/select.rst @@ -182,7 +182,7 @@ Returns the number of matching rows. .. code-block:: python - from piccolo.query.methods.select import Count + from piccolo.query.functions.aggregate import Count >> await Band.select(Count()).where(Band.popularity > 100) [{'count': 3}] @@ -196,7 +196,7 @@ Returns the average for a given column: .. code-block:: python - >>> from piccolo.query import Avg + >>> from piccolo.query.functions.aggregate import Avg >>> response = await Band.select(Avg(Band.popularity)).first() >>> response["avg"] 750.0 @@ -208,7 +208,7 @@ Returns the sum for a given column: .. code-block:: python - >>> from piccolo.query import Sum + >>> from piccolo.query.functions.aggregate import Sum >>> response = await Band.select(Sum(Band.popularity)).first() >>> response["sum"] 1500 @@ -220,7 +220,7 @@ Returns the maximum for a given column: .. code-block:: python - >>> from piccolo.query import Max + >>> from piccolo.query.functions.aggregate import Max >>> response = await Band.select(Max(Band.popularity)).first() >>> response["max"] 1000 @@ -232,7 +232,7 @@ Returns the minimum for a given column: .. code-block:: python - >>> from piccolo.query import Min + >>> from piccolo.query.functions.aggregate import Min >>> response = await Band.select(Min(Band.popularity)).first() >>> response["min"] 500 @@ -244,7 +244,7 @@ You also can have multiple different aggregate functions in one query: .. code-block:: python - >>> from piccolo.query import Avg, Sum + >>> from piccolo.query.functions.aggregate import Avg, Sum >>> response = await Band.select( ... Avg(Band.popularity), ... Sum(Band.popularity) diff --git a/piccolo/query/__init__.py b/piccolo/query/__init__.py index 000a47e76..2fcc2df7e 100644 --- a/piccolo/query/__init__.py +++ b/piccolo/query/__init__.py @@ -1,9 +1,9 @@ from piccolo.columns.combination import WhereRaw from .base import Query +from .functions.aggregate import Avg, Max, Min, Sum from .methods import ( Alter, - Avg, Count, Create, CreateIndex, @@ -11,12 +11,9 @@ DropIndex, Exists, Insert, - Max, - Min, Objects, Raw, Select, - Sum, TableExists, Update, ) diff --git a/piccolo/query/functions/__init__.py b/piccolo/query/functions/__init__.py index 6912a5ff3..d0195cc40 100644 --- a/piccolo/query/functions/__init__.py +++ b/piccolo/query/functions/__init__.py @@ -1,10 +1,16 @@ +from .aggregate import Avg, Count, Max, Min, Sum from .string import Length, Lower, Ltrim, Reverse, Rtrim, Upper __all__ = ( + "Avg", + "Count", "Length", "Lower", "Ltrim", + "Max", + "Min", "Reverse", "Rtrim", + "Sum", "Upper", ) diff --git a/piccolo/query/functions/aggregate.py b/piccolo/query/functions/aggregate.py new file mode 100644 index 000000000..61dd36a46 --- /dev/null +++ b/piccolo/query/functions/aggregate.py @@ -0,0 +1,179 @@ +import typing as t + +from piccolo.columns.base import Column +from piccolo.querystring import QueryString + +from .base import Function + + +class Avg(Function): + """ + ``AVG()`` SQL function. Column type must be numeric to run the query. + + .. code-block:: python + + await Band.select(Avg(Band.popularity)).run() + + # We can use an alias. These two are equivalent: + + await Band.select( + Avg(Band.popularity, alias="popularity_avg") + ).run() + + await Band.select( + Avg(Band.popularity).as_alias("popularity_avg") + ).run() + + """ + + function_name = "AVG" + + +class Count(QueryString): + """ + Used in ``Select`` queries, usually in conjunction with the ``group_by`` + clause:: + + >>> await Band.select( + ... Band.manager.name.as_alias('manager_name'), + ... Count(alias='band_count') + ... ).group_by(Band.manager) + [{'manager_name': 'Guido', 'count': 1}, ...] + + It can also be used without the ``group_by`` clause (though you may prefer + to the :meth:`Table.count ` method instead, as + it's more convenient):: + + >>> await Band.select(Count()) + [{'count': 3}] + + """ + + def __init__( + self, + column: t.Optional[Column] = None, + distinct: t.Optional[t.Sequence[Column]] = None, + alias: str = "count", + ): + """ + :param column: + If specified, the count is for non-null values in that column. + :param distinct: + If specified, the count is for distinct values in those columns. + :param alias: + The name of the value in the response:: + + # These two are equivalent: + + await Band.select( + Band.name, Count(alias="total") + ).group_by(Band.name) + + await Band.select( + Band.name, + Count().as_alias("total") + ).group_by(Band.name) + + """ + if distinct and column: + raise ValueError("Only specify `column` or `distinct`") + + if distinct: + engine_type = distinct[0]._meta.engine_type + if engine_type == "sqlite": + # SQLite doesn't allow us to specify multiple columns, so + # instead we concatenate the values. + column_names = " || ".join("{}" for _ in distinct) + else: + column_names = ", ".join("{}" for _ in distinct) + + return super().__init__( + f"COUNT(DISTINCT({column_names}))", *distinct, alias=alias + ) + else: + if column: + return super().__init__("COUNT({})", column, alias=alias) + else: + return super().__init__("COUNT(*)", alias=alias) + + +class Min(Function): + """ + ``MIN()`` SQL function. + + .. code-block:: python + + await Band.select(Min(Band.popularity)).run() + + # We can use an alias. These two are equivalent: + + await Band.select( + Min(Band.popularity, alias="popularity_min") + ).run() + + await Band.select( + Min(Band.popularity).as_alias("popularity_min") + ).run() + + """ + + function_name = "MIN" + + +class Max(Function): + """ + ``MAX()`` SQL function. + + .. code-block:: python + + await Band.select( + Max(Band.popularity) + ).run() + + # We can use an alias. These two are equivalent: + + await Band.select( + Max(Band.popularity, alias="popularity_max") + ).run() + + await Band.select( + Max(Band.popularity).as_alias("popularity_max") + ).run() + + """ + + function_name = "MAX" + + +class Sum(Function): + """ + ``SUM()`` SQL function. Column type must be numeric to run the query. + + .. code-block:: python + + await Band.select( + Sum(Band.popularity) + ).run() + + # We can use an alias. These two are equivalent: + + await Band.select( + Sum(Band.popularity, alias="popularity_sum") + ).run() + + await Band.select( + Sum(Band.popularity).as_alias("popularity_sum") + ).run() + + """ + + function_name = "SUM" + + +__all__ = ( + "Avg", + "Count", + "Min", + "Max", + "Sum", +) diff --git a/piccolo/query/methods/__init__.py b/piccolo/query/methods/__init__.py index 6c1854381..f4b9a59f1 100644 --- a/piccolo/query/methods/__init__.py +++ b/piccolo/query/methods/__init__.py @@ -9,6 +9,23 @@ from .objects import Objects from .raw import Raw from .refresh import Refresh -from .select import Avg, Max, Min, Select, Sum +from .select import Select from .table_exists import TableExists from .update import Update + +__all__ = ( + "Alter", + "Count", + "Create", + "CreateIndex", + "Delete", + "DropIndex", + "Exists", + "Insert", + "Objects", + "Raw", + "Refresh", + "Select", + "TableExists", + "Update", +) diff --git a/piccolo/query/methods/count.py b/piccolo/query/methods/count.py index fdd0972cf..3f9257bdc 100644 --- a/piccolo/query/methods/count.py +++ b/piccolo/query/methods/count.py @@ -4,7 +4,7 @@ from piccolo.custom_types import Combinable from piccolo.query.base import Query -from piccolo.query.methods.select import Count as SelectCount +from piccolo.query.functions.aggregate import Count as CountFunction from piccolo.query.mixins import WhereDelegate from piccolo.querystring import QueryString @@ -50,7 +50,7 @@ def default_querystrings(self) -> t.Sequence[QueryString]: table: t.Type[Table] = self.table query = table.select( - SelectCount(column=self.column, distinct=self._distinct) + CountFunction(column=self.column, distinct=self._distinct) ) query.where_delegate._where = self.where_delegate._where diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index 124b3ca6c..f5f0e3fa1 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -1,6 +1,5 @@ from __future__ import annotations -import decimal import itertools import typing as t from collections import OrderedDict @@ -36,13 +35,8 @@ from piccolo.custom_types import Combinable from piccolo.table import Table # noqa - -def is_numeric_column(column: Column) -> bool: - return column.value_type in (int, decimal.Decimal, float) - - -def is_string_column(column: Column) -> bool: - return column.value_type is str +# Here to avoid breaking changes - will be removed in the future. +from piccolo.query.functions.aggregate import Count # noqa: F401 class SelectRaw(Selectable): @@ -67,222 +61,6 @@ def get_select_string( return self.querystring -class Avg(Selectable): - """ - ``AVG()`` SQL function. Column type must be numeric to run the query. - - .. code-block:: python - - await Band.select(Avg(Band.popularity)).run() - - # We can use an alias. These two are equivalent: - - await Band.select( - Avg(Band.popularity, alias="popularity_avg") - ).run() - - await Band.select( - Avg(Band.popularity).as_alias("popularity_avg") - ).run() - - """ - - def __init__(self, column: Column, alias: str = "avg"): - if is_numeric_column(column): - self.column = column - else: - raise ValueError("Column type must be numeric to run the query.") - self._alias = alias - - def get_select_string( - self, engine_type: str, with_alias: bool = True - ) -> QueryString: - column_name = self.column._meta.get_full_name(with_alias=False) - return QueryString(f'AVG({column_name}) AS "{self._alias}"') - - -class Count(Selectable): - """ - Used in ``Select`` queries, usually in conjunction with the ``group_by`` - clause:: - - >>> await Band.select( - ... Band.manager.name.as_alias('manager_name'), - ... Count(alias='band_count') - ... ).group_by(Band.manager) - [{'manager_name': 'Guido', 'count': 1}, ...] - - It can also be used without the ``group_by`` clause (though you may prefer - to the :meth:`Table.count ` method instead, as - it's more convenient):: - - >>> await Band.select(Count()) - [{'count': 3}] - - """ - - def __init__( - self, - column: t.Optional[Column] = None, - distinct: t.Optional[t.Sequence[Column]] = None, - alias: str = "count", - ): - """ - :param column: - If specified, the count is for non-null values in that column. - :param distinct: - If specified, the count is for distinct values in those columns. - :param alias: - The name of the value in the response:: - - # These two are equivalent: - - await Band.select( - Band.name, Count(alias="total") - ).group_by(Band.name) - - await Band.select( - Band.name, - Count().as_alias("total") - ).group_by(Band.name) - - """ - if distinct and column: - raise ValueError("Only specify `column` or `distinct`") - - self.column = column - self.distinct = distinct - self._alias = alias - - def get_select_string( - self, engine_type: str, with_alias: bool = True - ) -> QueryString: - expression: str - - if self.distinct: - if engine_type == "sqlite": - # SQLite doesn't allow us to specify multiple columns, so - # instead we concatenate the values. - column_names = " || ".join( - i._meta.get_full_name(with_alias=False) - for i in self.distinct - ) - else: - column_names = ", ".join( - i._meta.get_full_name(with_alias=False) - for i in self.distinct - ) - - expression = f"DISTINCT ({column_names})" - else: - if self.column: - expression = self.column._meta.get_full_name(with_alias=False) - else: - expression = "*" - - return QueryString(f'COUNT({expression}) AS "{self._alias}"') - - -class Max(Selectable): - """ - ``MAX()`` SQL function. - - .. code-block:: python - - await Band.select( - Max(Band.popularity) - ).run() - - # We can use an alias. These two are equivalent: - - await Band.select( - Max(Band.popularity, alias="popularity_max") - ).run() - - await Band.select( - Max(Band.popularity).as_alias("popularity_max") - ).run() - - """ - - def __init__(self, column: Column, alias: str = "max"): - self.column = column - self._alias = alias - - def get_select_string( - self, engine_type: str, with_alias: bool = True - ) -> QueryString: - column_name = self.column._meta.get_full_name(with_alias=False) - return QueryString(f'MAX({column_name}) AS "{self._alias}"') - - -class Min(Selectable): - """ - ``MIN()`` SQL function. - - .. code-block:: python - - await Band.select(Min(Band.popularity)).run() - - # We can use an alias. These two are equivalent: - - await Band.select( - Min(Band.popularity, alias="popularity_min") - ).run() - - await Band.select( - Min(Band.popularity).as_alias("popularity_min") - ).run() - - """ - - def __init__(self, column: Column, alias: str = "min"): - self.column = column - self._alias = alias - - def get_select_string( - self, engine_type: str, with_alias: bool = True - ) -> QueryString: - column_name = self.column._meta.get_full_name(with_alias=False) - return QueryString(f'MIN({column_name}) AS "{self._alias}"') - - -class Sum(Selectable): - """ - ``SUM()`` SQL function. Column type must be numeric to run the query. - - .. code-block:: python - - await Band.select( - Sum(Band.popularity) - ).run() - - # We can use an alias. These two are equivalent: - - await Band.select( - Sum(Band.popularity, alias="popularity_sum") - ).run() - - await Band.select( - Sum(Band.popularity).as_alias("popularity_sum") - ).run() - - """ - - def __init__(self, column: Column, alias: str = "sum"): - if is_numeric_column(column): - self.column = column - else: - raise ValueError("Column type must be numeric to run the query.") - self._alias = alias - - def get_select_string( - self, engine_type: str, with_alias: bool = True - ) -> QueryString: - column_name = self.column._meta.get_full_name(with_alias=False) - return QueryString(f'SUM({column_name}) AS "{self._alias}"') - - OptionalDict = t.Optional[t.Dict[str, t.Any]] diff --git a/tests/table/test_select.py b/tests/table/test_select.py index a2bb86981..ebf2c3ff8 100644 --- a/tests/table/test_select.py +++ b/tests/table/test_select.py @@ -7,7 +7,8 @@ from piccolo.columns import Date, Varchar from piccolo.columns.combination import WhereRaw from piccolo.query import OrderByRaw -from piccolo.query.methods.select import Avg, Count, Max, Min, SelectRaw, Sum +from piccolo.query.functions.aggregate import Avg, Count, Max, Min, Sum +from piccolo.query.methods.select import SelectRaw from piccolo.query.mixins import DistinctOnError from piccolo.table import Table, create_db_tables_sync, drop_db_tables_sync from tests.base import ( @@ -927,14 +928,6 @@ def test_chain_different_functions_alias(self): self.assertEqual(float(response["popularity_avg"]), 1003.3333333333334) self.assertEqual(response["popularity_sum"], 3010) - def test_avg_validation(self): - with self.assertRaises(ValueError): - Band.select(Avg(Band.name)).run_sync() - - def test_sum_validation(self): - with self.assertRaises(ValueError): - Band.select(Sum(Band.name)).run_sync() - def test_columns(self): """ Make sure the colums method can be used to specify which columns to From d495c8aa4ae856813f38a9d21a545d0bc3c8f05f Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 30 May 2024 00:10:20 +0100 Subject: [PATCH 37/39] make sure where clauses work with functions --- piccolo/custom_types.py | 3 ++- piccolo/query/mixins.py | 2 ++ .../test_string.py => test_functions.py} | 26 +++++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) rename tests/query/{functions/test_string.py => test_functions.py} (77%) diff --git a/piccolo/custom_types.py b/piccolo/custom_types.py index 2101bc689..74093afd9 100644 --- a/piccolo/custom_types.py +++ b/piccolo/custom_types.py @@ -4,10 +4,11 @@ if t.TYPE_CHECKING: # pragma: no cover from piccolo.columns.combination import And, Or, Where, WhereRaw # noqa + from piccolo.querystring import QueryString from piccolo.table import Table -Combinable = t.Union["Where", "WhereRaw", "And", "Or"] +Combinable = t.Union["Where", "WhereRaw", "And", "Or", "QueryString"] Iterable = t.Iterable[t.Any] diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index b64afb206..c8a2ce9e4 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -255,6 +255,8 @@ def _extract_columns(self, combinable: Combinable): elif isinstance(combinable, (And, Or)): self._extract_columns(combinable.first) self._extract_columns(combinable.second) + elif isinstance(combinable, WhereRaw): + self._where_columns.extend(combinable.querystring.columns) def where(self, *where: Combinable): for arg in where: diff --git a/tests/query/functions/test_string.py b/tests/query/test_functions.py similarity index 77% rename from tests/query/functions/test_string.py rename to tests/query/test_functions.py index bae565ceb..abe9a5f01 100644 --- a/tests/query/functions/test_string.py +++ b/tests/query/test_functions.py @@ -74,3 +74,29 @@ def test_nested_within_querystring(self): ).run_sync() self.assertListEqual(response, [{"concat": "GUIDO!"}]) + + +class TestWhereClause(FunctionTest): + + def test_where(self): + """ + Make sure where clauses work with functions. + """ + response = ( + Band.select(Band.name) + .where(Upper(Band.name) == "PYTHONISTAS") + .run_sync() + ) + self.assertListEqual(response, [{"name": "Pythonistas"}]) + + def test_where_with_joined_column(self): + """ + Make sure where clauses work with functions, when a joined column is + used. + """ + response = ( + Band.select(Band.name) + .where(Upper(Band.manager._.name) == "GUIDO") + .run_sync() + ) + self.assertListEqual(response, [{"name": "Pythonistas"}]) From 9fe57bedd90475c9c71010f605ce75f31e7d9d15 Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 30 May 2024 00:20:52 +0100 Subject: [PATCH 38/39] fix linter errors --- docs/src/piccolo/query_clauses/group_by.rst | 2 +- piccolo/custom_types.py | 3 +-- piccolo/query/methods/count.py | 2 +- piccolo/query/methods/delete.py | 2 +- piccolo/query/methods/exists.py | 2 +- piccolo/query/methods/objects.py | 2 +- piccolo/query/methods/select.py | 2 +- piccolo/query/methods/update.py | 2 +- piccolo/query/mixins.py | 2 +- 9 files changed, 9 insertions(+), 10 deletions(-) diff --git a/docs/src/piccolo/query_clauses/group_by.rst b/docs/src/piccolo/query_clauses/group_by.rst index 738c142b8..516a4ac00 100644 --- a/docs/src/piccolo/query_clauses/group_by.rst +++ b/docs/src/piccolo/query_clauses/group_by.rst @@ -25,7 +25,7 @@ In the following query, we get a count of the number of bands per manager: ... Band.manager.name.as_alias('manager_name'), ... Count(alias='band_count') ... ).group_by( - ... Band.manager + ... Band.manager.name ... ) [ diff --git a/piccolo/custom_types.py b/piccolo/custom_types.py index 74093afd9..2101bc689 100644 --- a/piccolo/custom_types.py +++ b/piccolo/custom_types.py @@ -4,11 +4,10 @@ if t.TYPE_CHECKING: # pragma: no cover from piccolo.columns.combination import And, Or, Where, WhereRaw # noqa - from piccolo.querystring import QueryString from piccolo.table import Table -Combinable = t.Union["Where", "WhereRaw", "And", "Or", "QueryString"] +Combinable = t.Union["Where", "WhereRaw", "And", "Or"] Iterable = t.Iterable[t.Any] diff --git a/piccolo/query/methods/count.py b/piccolo/query/methods/count.py index 3f9257bdc..99d46c39b 100644 --- a/piccolo/query/methods/count.py +++ b/piccolo/query/methods/count.py @@ -32,7 +32,7 @@ def __init__( ########################################################################### # Clauses - def where(self: Self, *where: Combinable) -> Self: + def where(self: Self, *where: t.Union[Combinable, QueryString]) -> Self: self.where_delegate.where(*where) return self diff --git a/piccolo/query/methods/delete.py b/piccolo/query/methods/delete.py index bc0746063..628b89b8e 100644 --- a/piccolo/query/methods/delete.py +++ b/piccolo/query/methods/delete.py @@ -30,7 +30,7 @@ def __init__(self, table: t.Type[Table], force: bool = False, **kwargs): self.returning_delegate = ReturningDelegate() self.where_delegate = WhereDelegate() - def where(self: Self, *where: Combinable) -> Self: + def where(self: Self, *where: t.Union[Combinable, QueryString]) -> Self: self.where_delegate.where(*where) return self diff --git a/piccolo/query/methods/exists.py b/piccolo/query/methods/exists.py index 26d25e03e..7fac83a75 100644 --- a/piccolo/query/methods/exists.py +++ b/piccolo/query/methods/exists.py @@ -16,7 +16,7 @@ def __init__(self, table: t.Type[TableInstance], **kwargs): super().__init__(table, **kwargs) self.where_delegate = WhereDelegate() - def where(self: Self, *where: Combinable) -> Self: + def where(self: Self, *where: t.Union[Combinable, QueryString]) -> Self: self.where_delegate.where(*where) return self diff --git a/piccolo/query/methods/objects.py b/piccolo/query/methods/objects.py index 7b8c3ad43..f11f78e8e 100644 --- a/piccolo/query/methods/objects.py +++ b/piccolo/query/methods/objects.py @@ -262,7 +262,7 @@ def order_by( self.order_by_delegate.order_by(*_columns, ascending=ascending) return self - def where(self: Self, *where: Combinable) -> Self: + def where(self: Self, *where: t.Union[Combinable, QueryString]) -> Self: self.where_delegate.where(*where) return self diff --git a/piccolo/query/methods/select.py b/piccolo/query/methods/select.py index f5f0e3fa1..fdb929f8a 100644 --- a/piccolo/query/methods/select.py +++ b/piccolo/query/methods/select.py @@ -427,7 +427,7 @@ def callback( self.callback_delegate.callback(callbacks, on=on) return self - def where(self: Self, *where: Combinable) -> Self: + def where(self: Self, *where: t.Union[Combinable, QueryString]) -> Self: self.where_delegate.where(*where) return self diff --git a/piccolo/query/methods/update.py b/piccolo/query/methods/update.py index ff6a10589..f75854c43 100644 --- a/piccolo/query/methods/update.py +++ b/piccolo/query/methods/update.py @@ -50,7 +50,7 @@ def values( self.values_delegate.values(values) return self - def where(self, *where: Combinable) -> Update: + def where(self, *where: t.Union[Combinable, QueryString]) -> Update: self.where_delegate.where(*where) return self diff --git a/piccolo/query/mixins.py b/piccolo/query/mixins.py index c8a2ce9e4..214d1b8d7 100644 --- a/piccolo/query/mixins.py +++ b/piccolo/query/mixins.py @@ -258,7 +258,7 @@ def _extract_columns(self, combinable: Combinable): elif isinstance(combinable, WhereRaw): self._where_columns.extend(combinable.querystring.columns) - def where(self, *where: Combinable): + def where(self, *where: t.Union[Combinable, QueryString]): for arg in where: if isinstance(arg, bool): raise ValueError( From 588d026b0b3206b7719b7fecaabec3b1ece00aaa Mon Sep 17 00:00:00 2001 From: Daniel Townsend Date: Thu, 30 May 2024 00:32:37 +0100 Subject: [PATCH 39/39] update docs --- docs/src/piccolo/query_types/select.rst | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/src/piccolo/query_types/select.rst b/docs/src/piccolo/query_types/select.rst index 053becd10..092291e4c 100644 --- a/docs/src/piccolo/query_types/select.rst +++ b/docs/src/piccolo/query_types/select.rst @@ -165,6 +165,31 @@ convenient. ------------------------------------------------------------------------------- +String functions +---------------- + +Piccolo has lots of string functions built-in. See +``piccolo/query/functions/string.py``. Here's an example using ``Upper``, to +convert values to uppercase: + +.. code-block:: python + + from piccolo.query.functions.string import Upper + + >> await Band.select(Upper(Band.name, alias='name')) + [{'name': 'PYTHONISTAS'}, ...] + +You can also use these within where clauses: + +.. code-block:: python + + from piccolo.query.functions.string import Upper + + >> await Band.select(Band.name).where(Upper(Band.manager.name) == 'GUIDO') + [{'name': 'Pythonistas'}] + +------------------------------------------------------------------------------- + .. _AggregateFunctions: Aggregate functions