From 022f9a70c1b2bc2fb291e85705fbbde89d14165f Mon Sep 17 00:00:00 2001 From: judynah Date: Tue, 29 Oct 2024 11:15:30 +0100 Subject: [PATCH 01/11] =?UTF-8?q?=F0=9F=9A=80=20Added=20tm1=20source?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/viadot/sources/__init__.py | 1 + src/viadot/sources/tm1.py | 187 +++++++++++++++++++++++++++++++++ 2 files changed, 188 insertions(+) create mode 100644 src/viadot/sources/tm1.py diff --git a/src/viadot/sources/__init__.py b/src/viadot/sources/__init__.py index 453d301a5..ed8fc0ae0 100644 --- a/src/viadot/sources/__init__.py +++ b/src/viadot/sources/__init__.py @@ -23,6 +23,7 @@ from .sql_server import SQLServer from .sqlite import SQLite from .supermetrics import Supermetrics +from .tm1 import TM1 from .uk_carbon_intensity import UKCarbonIntensity from .vid_club import VidClub diff --git a/src/viadot/sources/tm1.py b/src/viadot/sources/tm1.py new file mode 100644 index 000000000..083c07705 --- /dev/null +++ b/src/viadot/sources/tm1.py @@ -0,0 +1,187 @@ +"""Source for connecting to TM1.""" + +from typing import Any, Literal + +from pydantic import BaseModel, SecretStr +from TM1py.Services import TM1Service +import pandas as pd + +from src.viadot.exceptions import ValidationError +from viadot.config import get_source_credentials +from viadot.sources.base import Source + + +class TM1Credentials(BaseModel): + """TM1 credentials. + + Uses simple authentication: + - username: The user name to use. + - password: The password to use. + - address: + - port: + """ + + username: str + password: SecretStr + address: str + port: str + + +class TM1(Source): + """Class for downloading data from TM1 Software using TM1py library.""" + + def __init__( + self, + credentials: dict[str, Any] | None = None, + config_key: str = "TM1", + mdx_query: str | None = None, + cube: str | None = None, + view: str | None = None, + dimension: str | None = None, + hierarchy: str | None = None, + limit: int | None = None, + private: bool = False, + verify: bool = False, + *args, + **kwargs, + ): + """C Creating an instance of TM1 source class. To download the data to the dataframe user needs to specify MDX query or + combination of cube and view. + + Args: + credentials (Dict[str, Any], optional): Credentials stored in a dictionary. Required credentials: username, + password, address, port. Defaults to None. + config_key (str, optional): Credential key to dictionary where credentials are stored. Defaults to "TM1". + mdx_query (str, optional): MDX select query needed to download the data. Defaults to None. + cube (str, optional): Cube name from which data will be downloaded. Defaults to None. + view (str, optional): View name from which data will be downloaded. Defaults to None. + dimension (str, optional): Dimension name. Defaults to None. + hierarchy (str, optional): Hierarchy name. Defaults to None. + limit (str, optional): How many rows should be extracted. If None all the available rows will + be downloaded. Defaults to None. + private (bool, optional): Whether or not data download shoulb be private. Defaults to False. + verify (bool, optional): Whether or not verify SSL certificates while. Defaults to False. + + + Raises: + CredentialError: When credentials are not found. + + """ + raw_creds = credentials or get_source_credentials(config_key) + validated_creds = dict(TM1Credentials(**raw_creds)) + + self.config_key = config_key + self.mdx_query = mdx_query + self.cube = cube + self.view = view + self.dimension = dimension + self.hierarchy = hierarchy + self.limit = limit + self.private = private + self.verify = verify + + super().__init__(*args, credentials=credentials, **kwargs) + + def get_connection(self) -> TM1Service: + """Start a connection to TM1 instance. + + Returns: + TM1Service: Service instance if connection is succesfull. + """ + return TM1Service( + address=self.credentials["address"], + port=self.credentials["port"], + user=self.credentials["username"], + password=self.credentials["password"], + ssl=self.verify, + ) + + def get_cubes_names(self) -> list: + """Get list of available cubes in TM1 instance. + + Returns: + list: List containing available cubes names. + + """ + conn = self.get_connection() + return conn.cubes.get_all_names() + + def get_views_names(self) -> list: + """Get list of available views in TM1 cube instance. + + Returns: + list: List containing available views names. + + """ + conn = self.get_connection() + return conn.views.get_all_names(self.cube) + + def get_dimensions_names(self) -> list: + """Get list of available dimensions in TM1 instance. + + Returns: + list: List containing available dimensions names. + + """ + conn = self.get_connection() + return conn.dimensions.get_all_names() + + def get_hierarchies_names(self) -> list: + """Get list of available hierarchies in TM1 dimension instance. + + Returns: + list: List containing available hierarchies names. + + """ + conn = self.get_connection() + return conn.hierarchies.get_all_names(self.dimension) + + def get_available_elements(self) -> list: + """Get list of available elements in TM1 instance based on hierarchy and diemension. + + Returns: + list: List containing available elements names. + + """ + conn = self.get_connection() + return conn.elements.get_element_names( + dimension_name=self.dimension, hierarchy_name=self.hierarchy + ) + + def to_df(self, if_empty: Literal["warn", "fail", "skip"] = "skip") -> pd.DataFrame: + """Function for downloading data from TM1 to pd.DataFrame. To download the data to the dataframe user needs to specify MDX query or + combination of cube and view. + + Args: + if_empty (Literal["warn", "fail", "skip"], optional): What to do if output DataFrame is empty. Defaults to "skip". + + Returns: + pd.DataFrame: DataFrame with data downloaded from TM1 view. + + Raises: + ValidationError: When mdx and cube + view are not specified or when combination of both is specified. + """ + conn = self.get_connection() + + if self.mdx_query is None and (self.cube is None or self.view is None): + raise ValidationError("MDX query or cube and view are required.") + if self.mdx_query is not None and ( + self.cube is not None or self.view is not None + ): + raise ValidationError("Specify only one: MDX query or cube and view.") + if self.cube is not None and self.view is not None: + df = conn.cubes.cells.execute_view_dataframe( + cube_name=self.cube, + view_name=self.view, + private=self.private, + top=self.limit, + ) + elif self.mdx_query is not None: + df = conn.cubes.cells.execute_mdx_dataframe(self.mdx_query) + + self.logger.info( + f"Data was successfully transformed into DataFrame: {len(df.columns)} columns and {len(df)} rows." + ) + if df.empty is True: + self._handle_if_empty(if_empty) + return df From 3216aff41f0dca1cbb8f91a8140c3ee7f4b2db1e Mon Sep 17 00:00:00 2001 From: judynah Date: Fri, 15 Nov 2024 16:37:51 +0100 Subject: [PATCH 02/11] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20Updated=20tm1=20sour?= =?UTF-8?q?ce?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/viadot/sources/tm1.py | 73 +++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 29 deletions(-) diff --git a/src/viadot/sources/tm1.py b/src/viadot/sources/tm1.py index 083c07705..402409e60 100644 --- a/src/viadot/sources/tm1.py +++ b/src/viadot/sources/tm1.py @@ -2,9 +2,9 @@ from typing import Any, Literal +import pandas as pd from pydantic import BaseModel, SecretStr from TM1py.Services import TM1Service -import pandas as pd from src.viadot.exceptions import ValidationError from viadot.config import get_source_credentials @@ -17,8 +17,8 @@ class TM1Credentials(BaseModel): Uses simple authentication: - username: The user name to use. - password: The password to use. - - address: - - port: + - address: The ip adress to use. + - port: The port """ username: str @@ -45,30 +45,37 @@ def __init__( *args, **kwargs, ): - """C Creating an instance of TM1 source class. To download the data to the dataframe user needs to specify MDX query or + """Creating an instance of TM1 source class. + + To download the data to the dataframe user needs to specify MDX query or combination of cube and view. Args: - credentials (Dict[str, Any], optional): Credentials stored in a dictionary. Required credentials: username, - password, address, port. Defaults to None. - config_key (str, optional): Credential key to dictionary where credentials are stored. Defaults to "TM1". - mdx_query (str, optional): MDX select query needed to download the data. Defaults to None. - cube (str, optional): Cube name from which data will be downloaded. Defaults to None. - view (str, optional): View name from which data will be downloaded. Defaults to None. + credentials (Dict[str, Any], optional): Credentials stored in a dictionary. + Required credentials: username, password, address, port. Defaults to None. + config_key (str, optional): Credential key to dictionary where credentials + are stored. Defaults to "TM1". + mdx_query (str, optional): MDX select query needed to download the data. + Defaults to None. + cube (str, optional): Cube name from which data will be downloaded. + Defaults to None. + view (str, optional): View name from which data will be downloaded. + Defaults to None. dimension (str, optional): Dimension name. Defaults to None. hierarchy (str, optional): Hierarchy name. Defaults to None. - limit (str, optional): How many rows should be extracted. If None all the available rows will - be downloaded. Defaults to None. - private (bool, optional): Whether or not data download shoulb be private. Defaults to False. - verify (bool, optional): Whether or not verify SSL certificates while. Defaults to False. - + limit (str, optional): How many rows should be extracted. + If None all the available rows will be downloaded. Defaults to None. + private (bool, optional): Whether or not data download should be private. + Defaults to False. + verify (bool, optional): Whether or not verify SSL certificates while. + Defaults to False. Raises: CredentialError: When credentials are not found. - + """ raw_creds = credentials or get_source_credentials(config_key) - validated_creds = dict(TM1Credentials(**raw_creds)) + self.validated_creds = dict(TM1Credentials(**raw_creds)) self.config_key = config_key self.mdx_query = mdx_query @@ -80,7 +87,7 @@ def __init__( self.private = private self.verify = verify - super().__init__(*args, credentials=credentials, **kwargs) + super().__init__(*args, credentials=self.validated_creds, **kwargs) def get_connection(self) -> TM1Service: """Start a connection to TM1 instance. @@ -89,10 +96,10 @@ def get_connection(self) -> TM1Service: TM1Service: Service instance if connection is succesfull. """ return TM1Service( - address=self.credentials["address"], - port=self.credentials["port"], - user=self.credentials["username"], - password=self.credentials["password"], + address=self.validated_creds.get("address"), + port=self.validated_creds.get("port"), + user=self.validated_creds.get("username"), + password=self.validated_creds.get("password").get_secret_value(), ssl=self.verify, ) @@ -137,7 +144,9 @@ def get_hierarchies_names(self) -> list: return conn.hierarchies.get_all_names(self.dimension) def get_available_elements(self) -> list: - """Get list of available elements in TM1 instance based on hierarchy and diemension. + """Get list of available elements in TM1 instance. + + Elements are extracted based on hierarchy and dimension. Returns: list: List containing available elements names. @@ -149,26 +158,32 @@ def get_available_elements(self) -> list: ) def to_df(self, if_empty: Literal["warn", "fail", "skip"] = "skip") -> pd.DataFrame: - """Function for downloading data from TM1 to pd.DataFrame. To download the data to the dataframe user needs to specify MDX query or - combination of cube and view. + """Function for downloading data from TM1 to pd.DataFrame. + + To download the data to the dataframe user needs to specify MDX query + or combination of cube and view. Args: - if_empty (Literal["warn", "fail", "skip"], optional): What to do if output DataFrame is empty. Defaults to "skip". + if_empty (Literal["warn", "fail", "skip"], optional): What to do if output + DataFrame is empty. Defaults to "skip". Returns: pd.DataFrame: DataFrame with data downloaded from TM1 view. Raises: - ValidationError: When mdx and cube + view are not specified or when combination of both is specified. + ValidationError: When mdx and cube + view are not specified + or when combination of both is specified. """ conn = self.get_connection() if self.mdx_query is None and (self.cube is None or self.view is None): - raise ValidationError("MDX query or cube and view are required.") + error_msg = "MDX query or cube and view are required." + raise ValidationError(error_msg) if self.mdx_query is not None and ( self.cube is not None or self.view is not None ): - raise ValidationError("Specify only one: MDX query or cube and view.") + error_msg="Specify only one: MDX query or cube and view." + raise ValidationError(error_msg) if self.cube is not None and self.view is not None: df = conn.cubes.cells.execute_view_dataframe( cube_name=self.cube, From 684e8f62516ee0eb52c9fa10488fdc91e573d25b Mon Sep 17 00:00:00 2001 From: judynah Date: Mon, 18 Nov 2024 10:55:01 +0100 Subject: [PATCH 03/11] =?UTF-8?q?=E2=9C=A8=20Added=20tm1=20task?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/viadot/orchestration/prefect/tasks/tm1.py | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 src/viadot/orchestration/prefect/tasks/tm1.py diff --git a/src/viadot/orchestration/prefect/tasks/tm1.py b/src/viadot/orchestration/prefect/tasks/tm1.py new file mode 100644 index 000000000..d77048587 --- /dev/null +++ b/src/viadot/orchestration/prefect/tasks/tm1.py @@ -0,0 +1,80 @@ +"""Task for downloading data from TM1 to a pandas DataFrame.""" + +from typing import Any + +from pandas import DataFrame +from prefect.logging import get_run_logger + +from src.viadot.config import get_source_credentials +from src.viadot.orchestration.prefect import tasks +from src.viadot.orchestration.prefect.exceptions import MissingSourceCredentialsError +from src.viadot.orchestration.prefect.utils import get_credentials +from src.viadot.sources.tm1 import TM1 + + +@tasks(retries=3, retry_delay_seconds=10, timeout_seconds=60 * 60 * 3) +def tm1_to_df( + mdx_query: str | None = None, + cube: str | None = None, + view: str | None = None, + limit: int | None = None, + private: bool = False, + credentials_secret: dict[str, Any] | None = None, + config_key: str = "TM1", + verify: bool = False, + if_empty: str = "skip", +) -> DataFrame: + """Download data from TM1 to pandas DataFrame. + + Args: + credentials_secret (dict[str, Any], optional): The name of the secret that + stores TM1 credentials. + More info on: https://docs.prefect.io/concepts/blocks/. Defaults to None. + config_key (str, optional): The key in the viadot config holding relevant + credentials. Defaults to "TM1". + mdx_query (str, optional): MDX select query needed to download the data. + Defaults to None. + cube (str, optional): Cube name from which data will be downloaded. + Defaults to None. + view (str, optional): View name from which data will be downloaded. + Defaults to None. + limit (str, optional): How many rows should be extracted. + If None all the available rows will be downloaded. Defaults to None. + private (bool, optional): Whether or not data download should be private. + Defaults to False. + verify (bool, optional): Whether or not verify SSL certificates. + Defaults to False. + if_empty (str, optional): What to do if output DataFrame is empty. + Defaults to "skip". + + """ + if not (credentials_secret or config_key): + raise MissingSourceCredentialsError + + logger = get_run_logger() + + credentials = get_source_credentials(config_key) or get_credentials( + credentials_secret + ) + + bc = TM1( + credentials=credentials, + config_key=config_key, + mdx_query=mdx_query, + cube=cube, + view=view, + limit=limit, + private=private, + verify=verify, + ) + + df = bc.to_df(if_empty=if_empty) + + nrows = df.shape[0] + ncols = df.shape[1] + + logger.info( + f"Successfully downloaded {nrows} rows and {ncols} columns of data to a DataFrame." + ) + + return df From bf152d19eaafac53f26fac76007f58b04fa4f93b Mon Sep 17 00:00:00 2001 From: judynah Date: Mon, 18 Nov 2024 13:32:36 +0100 Subject: [PATCH 04/11] =?UTF-8?q?=E2=9C=A8=20Added=20tm1=20prefect=20flow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../orchestration/prefect/flows/__init__.py | 2 + .../prefect/flows/tm1_to_parquet.py | 68 +++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 src/viadot/orchestration/prefect/flows/tm1_to_parquet.py diff --git a/src/viadot/orchestration/prefect/flows/__init__.py b/src/viadot/orchestration/prefect/flows/__init__.py index c89469388..07cb8fbd7 100644 --- a/src/viadot/orchestration/prefect/flows/__init__.py +++ b/src/viadot/orchestration/prefect/flows/__init__.py @@ -32,6 +32,7 @@ from .sql_server_to_parquet import sql_server_to_parquet from .sql_server_transform import sql_server_transform from .supermetrics_to_adls import supermetrics_to_adls +from .tm1_to_parquet import tm1_to_parquet from .transform import transform from .transform_and_catalog import transform_and_catalog from .vid_club_to_adls import vid_club_to_adls @@ -70,6 +71,7 @@ "sql_server_to_parquet", "sql_server_transform", "supermetrics_to_adls", + "tm1_to_parquet", "transform", "transform_and_catalog", "vid_club_to_adls", diff --git a/src/viadot/orchestration/prefect/flows/tm1_to_parquet.py b/src/viadot/orchestration/prefect/flows/tm1_to_parquet.py new file mode 100644 index 000000000..fa8216c7f --- /dev/null +++ b/src/viadot/orchestration/prefect/flows/tm1_to_parquet.py @@ -0,0 +1,68 @@ +"""Flow for downloading data from TM1 to a Parquet file.""" + +from typing import Literal + +from prefect import flow + +from viadot.orchestration.prefect.tasks.task_utils import df_to_parquet +from viadot.orchestration.prefect.tasks.tm1 import tm1_to_df + + +@flow( + name="extract--tm1--parquet", + description="Extract data from TM1 and load it into Parquet file", + retries=1, + retry_delay_seconds=60, +) +def tm1_to_parquet( # noqa: PLR0913 + path: str | None = None, + mdx_query: str | None = None, + cube: str | None = None, + view: str | None = None, + limit: int | None = None, + private: bool = False, + credentials_secret: str | None = None, + config_key: str | None = None, + if_empty: str = "skip", + if_exists: Literal["append", "replace", "skip"] = "replace", + verify: bool = True, +) -> None: + """Download data from TM1 to a Parquet file. + + Args: + mdx_query (str, optional): MDX select query needed to download the data. + Defaults to None. + cube (str, optional): Cube name from which data will be downloaded. + Defaults to None. + view (str, optional): View name from which data will be downloaded. + Defaults to None. + limit (str, optional): How many rows should be extracted. + If None all the available rows will be downloaded. Defaults to None. + private (bool, optional): Whether or not data download should be private. + Defaults to False. + credentials_secret (dict[str, Any], optional): The name of the secret that + stores TM1 credentials. + More info on: https://docs.prefect.io/concepts/blocks/. Defaults to None. + config_key (str, optional): The key in the viadot config holding relevant + credentials. Defaults to "TM1". + verify (bool, optional): Whether or not verify SSL certificates. + Defaults to False. + if_empty (str, optional): What to do if output DataFrame is empty. + Defaults to "skip". + """ + df = tm1_to_df( + mdx_query=mdx_query, + cube=cube, + view=view, + limit=limit, + private=private, + credentials_secret=credentials_secret, + config_key=config_key, + verify=verify, + if_empty=if_empty, + ) + return df_to_parquet( + df=df, + path=path, + if_exists=if_exists, + ) From 517de481fd48a558345fba8a697f2180b2b32397 Mon Sep 17 00:00:00 2001 From: judynah Date: Mon, 18 Nov 2024 13:34:48 +0100 Subject: [PATCH 05/11] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Refactor=20structure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../orchestration/prefect/tasks/__init__.py | 2 + src/viadot/orchestration/prefect/tasks/tm1.py | 41 +++++++++---------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/viadot/orchestration/prefect/tasks/__init__.py b/src/viadot/orchestration/prefect/tasks/__init__.py index bd9610f24..0bcfea102 100644 --- a/src/viadot/orchestration/prefect/tasks/__init__.py +++ b/src/viadot/orchestration/prefect/tasks/__init__.py @@ -30,6 +30,7 @@ from .sharepoint import sharepoint_download_file, sharepoint_to_df from .sql_server import create_sql_server_table, sql_server_query, sql_server_to_df from .supermetrics import supermetrics_to_df +from .tm1 import tm1_to_df from .vid_club import vid_club_to_df @@ -68,6 +69,7 @@ "sharepoint_to_df", "sql_server_query", "sql_server_to_df", + "tm1_to_df", "vid_club_to_df", "supermetrics_to_df", ] diff --git a/src/viadot/orchestration/prefect/tasks/tm1.py b/src/viadot/orchestration/prefect/tasks/tm1.py index d77048587..722afbdd0 100644 --- a/src/viadot/orchestration/prefect/tasks/tm1.py +++ b/src/viadot/orchestration/prefect/tasks/tm1.py @@ -1,51 +1,50 @@ """Task for downloading data from TM1 to a pandas DataFrame.""" -from typing import Any from pandas import DataFrame +from prefect import task from prefect.logging import get_run_logger -from src.viadot.config import get_source_credentials -from src.viadot.orchestration.prefect import tasks -from src.viadot.orchestration.prefect.exceptions import MissingSourceCredentialsError -from src.viadot.orchestration.prefect.utils import get_credentials -from src.viadot.sources.tm1 import TM1 +from viadot.config import get_source_credentials +from viadot.orchestration.prefect.exceptions import MissingSourceCredentialsError +from viadot.orchestration.prefect.utils import get_credentials +from viadot.sources.tm1 import TM1 -@tasks(retries=3, retry_delay_seconds=10, timeout_seconds=60 * 60 * 3) +@task(retries=3, retry_delay_seconds=10, timeout_seconds=60 * 60 * 3) def tm1_to_df( mdx_query: str | None = None, cube: str | None = None, view: str | None = None, limit: int | None = None, private: bool = False, - credentials_secret: dict[str, Any] | None = None, - config_key: str = "TM1", + credentials_secret: str | None = None, + config_key: str | None = None, verify: bool = False, if_empty: str = "skip", ) -> DataFrame: """Download data from TM1 to pandas DataFrame. Args: - credentials_secret (dict[str, Any], optional): The name of the secret that - stores TM1 credentials. - More info on: https://docs.prefect.io/concepts/blocks/. Defaults to None. - config_key (str, optional): The key in the viadot config holding relevant - credentials. Defaults to "TM1". mdx_query (str, optional): MDX select query needed to download the data. - Defaults to None. + Defaults to None. cube (str, optional): Cube name from which data will be downloaded. - Defaults to None. + Defaults to None. view (str, optional): View name from which data will be downloaded. - Defaults to None. + Defaults to None. limit (str, optional): How many rows should be extracted. - If None all the available rows will be downloaded. Defaults to None. + If None all the available rows will be downloaded. Defaults to None. private (bool, optional): Whether or not data download should be private. - Defaults to False. + Defaults to False. + credentials_secret (dict[str, Any], optional): The name of the secret that + stores TM1 credentials. + More info on: https://docs.prefect.io/concepts/blocks/. Defaults to None. + config_key (str, optional): The key in the viadot config holding relevant + credentials. Defaults to "TM1". verify (bool, optional): Whether or not verify SSL certificates. - Defaults to False. + Defaults to False. if_empty (str, optional): What to do if output DataFrame is empty. - Defaults to "skip". + Defaults to "skip". """ if not (credentials_secret or config_key): From 96045669cebc511cf9dfe04d2865ceb7e1966aa4 Mon Sep 17 00:00:00 2001 From: judynah Date: Mon, 25 Nov 2024 11:49:05 +0100 Subject: [PATCH 06/11] =?UTF-8?q?=E2=9C=85=20Add=20unit=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../prefect/flows/tm1_to_parquet.py | 6 +- src/viadot/orchestration/prefect/tasks/tm1.py | 2 +- src/viadot/sources/tm1.py | 44 +++++--- tests/unit/test_tm1.py | 106 ++++++++++++++++++ 4 files changed, 139 insertions(+), 19 deletions(-) create mode 100644 tests/unit/test_tm1.py diff --git a/src/viadot/orchestration/prefect/flows/tm1_to_parquet.py b/src/viadot/orchestration/prefect/flows/tm1_to_parquet.py index fa8216c7f..6976a514c 100644 --- a/src/viadot/orchestration/prefect/flows/tm1_to_parquet.py +++ b/src/viadot/orchestration/prefect/flows/tm1_to_parquet.py @@ -45,10 +45,12 @@ def tm1_to_parquet( # noqa: PLR0913 More info on: https://docs.prefect.io/concepts/blocks/. Defaults to None. config_key (str, optional): The key in the viadot config holding relevant credentials. Defaults to "TM1". - verify (bool, optional): Whether or not verify SSL certificates. - Defaults to False. if_empty (str, optional): What to do if output DataFrame is empty. Defaults to "skip". + if_exists (Literal["append", "replace", "skip"], optional): + What to do if the table exists. Defaults to "replace". + verify (bool, optional): Whether or not verify SSL certificate. + Defaults to False. """ df = tm1_to_df( mdx_query=mdx_query, diff --git a/src/viadot/orchestration/prefect/tasks/tm1.py b/src/viadot/orchestration/prefect/tasks/tm1.py index 722afbdd0..aa23b432e 100644 --- a/src/viadot/orchestration/prefect/tasks/tm1.py +++ b/src/viadot/orchestration/prefect/tasks/tm1.py @@ -41,7 +41,7 @@ def tm1_to_df( More info on: https://docs.prefect.io/concepts/blocks/. Defaults to None. config_key (str, optional): The key in the viadot config holding relevant credentials. Defaults to "TM1". - verify (bool, optional): Whether or not verify SSL certificates. + verify (bool, optional): Whether or not verify SSL certificate. Defaults to False. if_empty (str, optional): What to do if output DataFrame is empty. Defaults to "skip". diff --git a/src/viadot/sources/tm1.py b/src/viadot/sources/tm1.py index 402409e60..153b6e79d 100644 --- a/src/viadot/sources/tm1.py +++ b/src/viadot/sources/tm1.py @@ -6,8 +6,8 @@ from pydantic import BaseModel, SecretStr from TM1py.Services import TM1Service -from src.viadot.exceptions import ValidationError from viadot.config import get_source_credentials +from viadot.exceptions import ValidationError from viadot.sources.base import Source @@ -15,10 +15,10 @@ class TM1Credentials(BaseModel): """TM1 credentials. Uses simple authentication: - - username: The user name to use. - - password: The password to use. - - address: The ip adress to use. - - port: The port + - username: The user name to use for the connection. + - password: The password to use for the connection. + - address: The ip address to use for the connection. + - port: The port to use for the connection. """ username: str @@ -51,24 +51,25 @@ def __init__( combination of cube and view. Args: - credentials (Dict[str, Any], optional): Credentials stored in a dictionary. - Required credentials: username, password, address, port. Defaults to None. + credentials (dict[str, Any], optional): Credentials stored in a dictionary. + Required credentials: username, password, address, port. + Defaults to None. config_key (str, optional): Credential key to dictionary where credentials - are stored. Defaults to "TM1". + are stored. Defaults to "TM1". mdx_query (str, optional): MDX select query needed to download the data. - Defaults to None. + Defaults to None. cube (str, optional): Cube name from which data will be downloaded. - Defaults to None. + Defaults to None. view (str, optional): View name from which data will be downloaded. - Defaults to None. + Defaults to None. dimension (str, optional): Dimension name. Defaults to None. hierarchy (str, optional): Hierarchy name. Defaults to None. limit (str, optional): How many rows should be extracted. - If None all the available rows will be downloaded. Defaults to None. + If None all the available rows will be downloaded. Defaults to None. private (bool, optional): Whether or not data download should be private. - Defaults to False. - verify (bool, optional): Whether or not verify SSL certificates while. - Defaults to False. + Defaults to False. + verify (bool, optional): Whether or not verify SSL certificates. + Defaults to False. Raises: CredentialError: When credentials are not found. @@ -93,7 +94,7 @@ def get_connection(self) -> TM1Service: """Start a connection to TM1 instance. Returns: - TM1Service: Service instance if connection is succesfull. + TM1Service: Service instance if connection is successful. """ return TM1Service( address=self.validated_creds.get("address"), @@ -120,6 +121,9 @@ def get_views_names(self) -> list: list: List containing available views names. """ + if self.cube is None: + msg = "Missing cube name." + raise ValidationError(msg) conn = self.get_connection() return conn.views.get_all_names(self.cube) @@ -140,6 +144,10 @@ def get_hierarchies_names(self) -> list: list: List containing available hierarchies names. """ + if self.dimension is None: + msg = "Missing dimension name." + raise ValidationError(msg) + conn = self.get_connection() return conn.hierarchies.get_all_names(self.dimension) @@ -152,6 +160,10 @@ def get_available_elements(self) -> list: list: List containing available elements names. """ + if (self.dimension or self.hierarchy) is None: + msg = "Missing dimension or hierarchy." + raise ValidationError(msg) + conn = self.get_connection() return conn.elements.get_element_names( dimension_name=self.dimension, hierarchy_name=self.hierarchy diff --git a/tests/unit/test_tm1.py b/tests/unit/test_tm1.py new file mode 100644 index 000000000..0569cc5e2 --- /dev/null +++ b/tests/unit/test_tm1.py @@ -0,0 +1,106 @@ +from unittest.mock import patch + +import pandas as pd +import pytest + +from viadot.exceptions import ValidationError +from viadot.sources import TM1 +from viadot.sources.tm1 import TM1Credentials + + +data = { + "a": [420, 380, 390], + "b": [50, 40, 45] +} + +@pytest.fixture +def tm1_credentials(): + return TM1Credentials( + username="test_user", + password="test_password", # pragma: allowlist secret # noqa: S106 + address = "localhost", + port = "12356", + ) + +@pytest.fixture +def tm1(tm1_credentials: TM1Credentials): + return TM1( + credentials={ + "username": tm1_credentials.username, + "password": tm1_credentials.password, + "address": tm1_credentials.address, + "port": tm1_credentials.port, + }, + verify = False, + cube = "test_cube", + view = "test_view", + dimension = "test_dim", + hierarchy = "test_hier", + ) + + +@pytest.fixture +def tm1_mock(tm1_credentials: TM1Credentials, mocker): + mocker.patch("viadot.sources.TM1.get_connection", return_value=True) + + return TM1( + credentials={ + "username": tm1_credentials.username, + "password": tm1_credentials.password, + "address": tm1_credentials.address, + "port": tm1_credentials.port, + + }, + verify=False, + ) + +def test_tm1_initialization(tm1_mock): + """Test that the TM1 object is initialized with the correct credentials.""" + assert tm1_mock.credentials.get("address") == "localhost" + assert tm1_mock.credentials.get("username") == "test_user" + assert ( + tm1_mock.credentials.get("password").get_secret_value() + == "test_password" # pragma: allowlist secret + ) + +def test_get_cubes_names(tm1): + with patch('viadot.sources.tm1.TM1Service') as mock: + instance = mock.return_value + instance.cubes.get_all_names.return_value = ['msg', 'cuve'] + result = tm1.get_cubes_names() + assert result == ['msg', 'cuve'] + +def test_get_views_names(tm1): + with patch('viadot.sources.tm1.TM1Service') as mock: + instance = mock.return_value + instance.views.get_all_names.return_value = ['view1', 'view1'] + result = tm1.get_views_names() + assert result == ['view1', 'view1'] + +def test_get_dimensions_names(tm1): + with patch('viadot.sources.tm1.TM1Service') as mock: + instance = mock.return_value + instance.dimensions.get_all_names.return_value = ['dim1', 'dim2'] + result = tm1.get_dimensions_names() + assert result == ['dim1', 'dim2'] + +def test_get_available_elements(tm1): + with patch('viadot.sources.tm1.TM1Service') as mock: + instance = mock.return_value + instance.elements.get_element_names.return_value = ['el1', 'el2'] + result = tm1.get_available_elements() + assert result == ['el1', 'el2'] + +def test_to_df(tm1): + with patch('viadot.sources.tm1.TM1Service') as mock: + instance = mock.return_value + instance.cubes.cells.execute_view_dataframe.return_value = pd.DataFrame(data) + result = tm1.to_df() + assert isinstance(result, pd.DataFrame) + assert not result.empty + assert len(result.columns) == 2 + +def test_to_df_fail(tm1_mock): + with pytest.raises(ValidationError) as excinfo: + tm1_mock.to_df() + assert str(excinfo.value) == "MDX query or cube and view are required." From 1c663bc9522f1ef5ae8e571996264af616f82b0a Mon Sep 17 00:00:00 2001 From: judynah Date: Mon, 25 Nov 2024 12:14:59 +0100 Subject: [PATCH 07/11] =?UTF-8?q?=F0=9F=93=9D=20Update=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/viadot/sources/tm1.py | 17 ++++++++--------- tests/unit/test_tm1.py | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/viadot/sources/tm1.py b/src/viadot/sources/tm1.py index 153b6e79d..72639822a 100644 --- a/src/viadot/sources/tm1.py +++ b/src/viadot/sources/tm1.py @@ -9,6 +9,7 @@ from viadot.config import get_source_credentials from viadot.exceptions import ValidationError from viadot.sources.base import Source +from viadot.utils import add_viadot_metadata_columns class TM1Credentials(BaseModel): @@ -28,7 +29,7 @@ class TM1Credentials(BaseModel): class TM1(Source): - """Class for downloading data from TM1 Software using TM1py library.""" + """TM1 connector.""" def __init__( self, @@ -45,17 +46,14 @@ def __init__( *args, **kwargs, ): - """Creating an instance of TM1 source class. - - To download the data to the dataframe user needs to specify MDX query or - combination of cube and view. + """Create a TM1 connector instance. Args: credentials (dict[str, Any], optional): Credentials stored in a dictionary. Required credentials: username, password, address, port. Defaults to None. - config_key (str, optional): Credential key to dictionary where credentials - are stored. Defaults to "TM1". + config_key (str, optional): The key in the viadot config holding relevant + credentials. Defaults to "TM1". mdx_query (str, optional): MDX select query needed to download the data. Defaults to None. cube (str, optional): Cube name from which data will be downloaded. @@ -169,8 +167,9 @@ def get_available_elements(self) -> list: dimension_name=self.dimension, hierarchy_name=self.hierarchy ) + @add_viadot_metadata_columns def to_df(self, if_empty: Literal["warn", "fail", "skip"] = "skip") -> pd.DataFrame: - """Function for downloading data from TM1 to pd.DataFrame. + """Download data into a pandas DataFrame. To download the data to the dataframe user needs to specify MDX query or combination of cube and view. @@ -180,7 +179,7 @@ def to_df(self, if_empty: Literal["warn", "fail", "skip"] = "skip") -> pd.DataFr DataFrame is empty. Defaults to "skip". Returns: - pd.DataFrame: DataFrame with data downloaded from TM1 view. + pd.DataFrame: DataFrame with the data. Raises: ValidationError: When mdx and cube + view are not specified diff --git a/tests/unit/test_tm1.py b/tests/unit/test_tm1.py index 0569cc5e2..3e4906316 100644 --- a/tests/unit/test_tm1.py +++ b/tests/unit/test_tm1.py @@ -98,7 +98,7 @@ def test_to_df(tm1): result = tm1.to_df() assert isinstance(result, pd.DataFrame) assert not result.empty - assert len(result.columns) == 2 + assert len(result.columns) == 4 def test_to_df_fail(tm1_mock): with pytest.raises(ValidationError) as excinfo: From 84027b2b981440e6a726c00980d2413c8d05bd02 Mon Sep 17 00:00:00 2001 From: judynah Date: Mon, 25 Nov 2024 12:28:13 +0100 Subject: [PATCH 08/11] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Refactor=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../prefect/flows/tm1_to_parquet.py | 2 +- src/viadot/sources/__init__.py | 1 + src/viadot/sources/tm1.py | 2 +- tests/unit/test_tm1.py | 45 +++++++++---------- 4 files changed, 24 insertions(+), 26 deletions(-) diff --git a/src/viadot/orchestration/prefect/flows/tm1_to_parquet.py b/src/viadot/orchestration/prefect/flows/tm1_to_parquet.py index 6976a514c..436f28009 100644 --- a/src/viadot/orchestration/prefect/flows/tm1_to_parquet.py +++ b/src/viadot/orchestration/prefect/flows/tm1_to_parquet.py @@ -14,7 +14,7 @@ retries=1, retry_delay_seconds=60, ) -def tm1_to_parquet( # noqa: PLR0913 +def tm1_to_parquet( # noqa: PLR0913 path: str | None = None, mdx_query: str | None = None, cube: str | None = None, diff --git a/src/viadot/sources/__init__.py b/src/viadot/sources/__init__.py index ed8fc0ae0..d816bf86c 100644 --- a/src/viadot/sources/__init__.py +++ b/src/viadot/sources/__init__.py @@ -49,6 +49,7 @@ "Sftp", "Sharepoint", "Supermetrics", + "TM1", "Trino", "UKCarbonIntensity", "VidClub", diff --git a/src/viadot/sources/tm1.py b/src/viadot/sources/tm1.py index 72639822a..460452572 100644 --- a/src/viadot/sources/tm1.py +++ b/src/viadot/sources/tm1.py @@ -193,7 +193,7 @@ def to_df(self, if_empty: Literal["warn", "fail", "skip"] = "skip") -> pd.DataFr if self.mdx_query is not None and ( self.cube is not None or self.view is not None ): - error_msg="Specify only one: MDX query or cube and view." + error_msg = "Specify only one: MDX query or cube and view." raise ValidationError(error_msg) if self.cube is not None and self.view is not None: df = conn.cubes.cells.execute_view_dataframe( diff --git a/tests/unit/test_tm1.py b/tests/unit/test_tm1.py index 3e4906316..2d0f877ad 100644 --- a/tests/unit/test_tm1.py +++ b/tests/unit/test_tm1.py @@ -8,18 +8,15 @@ from viadot.sources.tm1 import TM1Credentials -data = { - "a": [420, 380, 390], - "b": [50, 40, 45] -} +data = {"a": [420, 380, 390], "b": [50, 40, 45]} @pytest.fixture def tm1_credentials(): return TM1Credentials( username="test_user", password="test_password", # pragma: allowlist secret # noqa: S106 - address = "localhost", - port = "12356", + address="localhost", + port="12356", ) @pytest.fixture @@ -31,11 +28,11 @@ def tm1(tm1_credentials: TM1Credentials): "address": tm1_credentials.address, "port": tm1_credentials.port, }, - verify = False, - cube = "test_cube", - view = "test_view", - dimension = "test_dim", - hierarchy = "test_hier", + verify=False, + cube="test_cube", + view="test_view", + dimension="test_dim", + hierarchy="test_hier", ) @@ -64,35 +61,35 @@ def test_tm1_initialization(tm1_mock): ) def test_get_cubes_names(tm1): - with patch('viadot.sources.tm1.TM1Service') as mock: + with patch("viadot.sources.tm1.TM1Service") as mock: instance = mock.return_value - instance.cubes.get_all_names.return_value = ['msg', 'cuve'] + instance.cubes.get_all_names.return_value = ["msg", "cuve"] result = tm1.get_cubes_names() - assert result == ['msg', 'cuve'] + assert result == ["msg", "cuve"] def test_get_views_names(tm1): - with patch('viadot.sources.tm1.TM1Service') as mock: + with patch("viadot.sources.tm1.TM1Service") as mock: instance = mock.return_value - instance.views.get_all_names.return_value = ['view1', 'view1'] + instance.views.get_all_names.return_value = ["view1", "view1"] result = tm1.get_views_names() - assert result == ['view1', 'view1'] + assert result == ["view1", "view1"] def test_get_dimensions_names(tm1): - with patch('viadot.sources.tm1.TM1Service') as mock: + with patch("viadot.sources.tm1.TM1Service") as mock: instance = mock.return_value - instance.dimensions.get_all_names.return_value = ['dim1', 'dim2'] + instance.dimensions.get_all_names.return_value = ["dim1", "dim2"] result = tm1.get_dimensions_names() - assert result == ['dim1', 'dim2'] + assert result == ["dim1", "dim2"] def test_get_available_elements(tm1): - with patch('viadot.sources.tm1.TM1Service') as mock: + with patch("viadot.sources.tm1.TM1Service") as mock: instance = mock.return_value - instance.elements.get_element_names.return_value = ['el1', 'el2'] + instance.elements.get_element_names.return_value = ["el1", "el2"] result = tm1.get_available_elements() - assert result == ['el1', 'el2'] + assert result == ["el1", "el2"] def test_to_df(tm1): - with patch('viadot.sources.tm1.TM1Service') as mock: + with patch("viadot.sources.tm1.TM1Service") as mock: instance = mock.return_value instance.cubes.cells.execute_view_dataframe.return_value = pd.DataFrame(data) result = tm1.to_df() From d19e53fc0797a1951c6b973bd9496e9d554c3d97 Mon Sep 17 00:00:00 2001 From: judynah Date: Mon, 25 Nov 2024 12:34:20 +0100 Subject: [PATCH 09/11] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Refactor=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/viadot/orchestration/prefect/tasks/tm1.py | 1 - tests/unit/test_tm1.py | 10 +++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/viadot/orchestration/prefect/tasks/tm1.py b/src/viadot/orchestration/prefect/tasks/tm1.py index aa23b432e..66d814ff9 100644 --- a/src/viadot/orchestration/prefect/tasks/tm1.py +++ b/src/viadot/orchestration/prefect/tasks/tm1.py @@ -1,6 +1,5 @@ """Task for downloading data from TM1 to a pandas DataFrame.""" - from pandas import DataFrame from prefect import task from prefect.logging import get_run_logger diff --git a/tests/unit/test_tm1.py b/tests/unit/test_tm1.py index 2d0f877ad..822eb34f8 100644 --- a/tests/unit/test_tm1.py +++ b/tests/unit/test_tm1.py @@ -10,6 +10,7 @@ data = {"a": [420, 380, 390], "b": [50, 40, 45]} + @pytest.fixture def tm1_credentials(): return TM1Credentials( @@ -19,6 +20,7 @@ def tm1_credentials(): port="12356", ) + @pytest.fixture def tm1(tm1_credentials: TM1Credentials): return TM1( @@ -46,11 +48,11 @@ def tm1_mock(tm1_credentials: TM1Credentials, mocker): "password": tm1_credentials.password, "address": tm1_credentials.address, "port": tm1_credentials.port, - }, verify=False, ) + def test_tm1_initialization(tm1_mock): """Test that the TM1 object is initialized with the correct credentials.""" assert tm1_mock.credentials.get("address") == "localhost" @@ -60,6 +62,7 @@ def test_tm1_initialization(tm1_mock): == "test_password" # pragma: allowlist secret ) + def test_get_cubes_names(tm1): with patch("viadot.sources.tm1.TM1Service") as mock: instance = mock.return_value @@ -67,6 +70,7 @@ def test_get_cubes_names(tm1): result = tm1.get_cubes_names() assert result == ["msg", "cuve"] + def test_get_views_names(tm1): with patch("viadot.sources.tm1.TM1Service") as mock: instance = mock.return_value @@ -74,6 +78,7 @@ def test_get_views_names(tm1): result = tm1.get_views_names() assert result == ["view1", "view1"] + def test_get_dimensions_names(tm1): with patch("viadot.sources.tm1.TM1Service") as mock: instance = mock.return_value @@ -81,6 +86,7 @@ def test_get_dimensions_names(tm1): result = tm1.get_dimensions_names() assert result == ["dim1", "dim2"] + def test_get_available_elements(tm1): with patch("viadot.sources.tm1.TM1Service") as mock: instance = mock.return_value @@ -88,6 +94,7 @@ def test_get_available_elements(tm1): result = tm1.get_available_elements() assert result == ["el1", "el2"] + def test_to_df(tm1): with patch("viadot.sources.tm1.TM1Service") as mock: instance = mock.return_value @@ -97,6 +104,7 @@ def test_to_df(tm1): assert not result.empty assert len(result.columns) == 4 + def test_to_df_fail(tm1_mock): with pytest.raises(ValidationError) as excinfo: tm1_mock.to_df() From ee63f3a5b45de08550e741709a32120cd814d8cb Mon Sep 17 00:00:00 2001 From: judynah Date: Wed, 4 Dec 2024 16:55:04 +0100 Subject: [PATCH 10/11] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20Improved=20logic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/viadot/orchestration/prefect/tasks/tm1.py | 10 ++- src/viadot/sources/tm1.py | 88 +++++++++---------- tests/unit/test_tm1.py | 51 ++++------- 3 files changed, 67 insertions(+), 82 deletions(-) diff --git a/src/viadot/orchestration/prefect/tasks/tm1.py b/src/viadot/orchestration/prefect/tasks/tm1.py index 66d814ff9..a5d0c497c 100644 --- a/src/viadot/orchestration/prefect/tasks/tm1.py +++ b/src/viadot/orchestration/prefect/tasks/tm1.py @@ -58,15 +58,17 @@ def tm1_to_df( bc = TM1( credentials=credentials, config_key=config_key, - mdx_query=mdx_query, - cube=cube, - view=view, limit=limit, private=private, verify=verify, ) - df = bc.to_df(if_empty=if_empty) + df = bc.to_df( + if_empty=if_empty, + mdx_query=mdx_query, + cube=cube, + view=view, + ) nrows = df.shape[0] ncols = df.shape[1] diff --git a/src/viadot/sources/tm1.py b/src/viadot/sources/tm1.py index 460452572..5220a0b06 100644 --- a/src/viadot/sources/tm1.py +++ b/src/viadot/sources/tm1.py @@ -35,11 +35,6 @@ def __init__( self, credentials: dict[str, Any] | None = None, config_key: str = "TM1", - mdx_query: str | None = None, - cube: str | None = None, - view: str | None = None, - dimension: str | None = None, - hierarchy: str | None = None, limit: int | None = None, private: bool = False, verify: bool = False, @@ -74,19 +69,14 @@ def __init__( """ raw_creds = credentials or get_source_credentials(config_key) - self.validated_creds = dict(TM1Credentials(**raw_creds)) + validated_creds = dict(TM1Credentials(**raw_creds)) self.config_key = config_key - self.mdx_query = mdx_query - self.cube = cube - self.view = view - self.dimension = dimension - self.hierarchy = hierarchy self.limit = limit self.private = private self.verify = verify - super().__init__(*args, credentials=self.validated_creds, **kwargs) + super().__init__(*args, credentials=validated_creds, **kwargs) def get_connection(self) -> TM1Service: """Start a connection to TM1 instance. @@ -95,10 +85,10 @@ def get_connection(self) -> TM1Service: TM1Service: Service instance if connection is successful. """ return TM1Service( - address=self.validated_creds.get("address"), - port=self.validated_creds.get("port"), - user=self.validated_creds.get("username"), - password=self.validated_creds.get("password").get_secret_value(), + address=self.credentials.get("address"), + port=self.credentials.get("port"), + user=self.credentials.get("username"), + password=self.credentials.get("password").get_secret_value(), ssl=self.verify, ) @@ -112,18 +102,18 @@ def get_cubes_names(self) -> list: conn = self.get_connection() return conn.cubes.get_all_names() - def get_views_names(self) -> list: + def get_views_names(self, cube: str) -> list: """Get list of available views in TM1 cube instance. + Args: + cube (str): Cube name. + Returns: list: List containing available views names. """ - if self.cube is None: - msg = "Missing cube name." - raise ValidationError(msg) conn = self.get_connection() - return conn.views.get_all_names(self.cube) + return conn.views.get_all_names(cube) def get_dimensions_names(self) -> list: """Get list of available dimensions in TM1 instance. @@ -135,79 +125,89 @@ def get_dimensions_names(self) -> list: conn = self.get_connection() return conn.dimensions.get_all_names() - def get_hierarchies_names(self) -> list: + def get_hierarchies_names(self, dimension: str) -> list: """Get list of available hierarchies in TM1 dimension instance. + Args: + dimension (str): Dimension name. + Returns: list: List containing available hierarchies names. """ - if self.dimension is None: - msg = "Missing dimension name." - raise ValidationError(msg) - conn = self.get_connection() - return conn.hierarchies.get_all_names(self.dimension) + return conn.hierarchies.get_all_names(dimension) - def get_available_elements(self) -> list: + def get_available_elements(self, dimension: str, hierarchy: str) -> list: """Get list of available elements in TM1 instance. Elements are extracted based on hierarchy and dimension. + Args: + dimension (str): Dimension name. + hierarchy (str): Hierarchy name. + Returns: list: List containing available elements names. """ - if (self.dimension or self.hierarchy) is None: + if dimension is None or hierarchy is None: msg = "Missing dimension or hierarchy." raise ValidationError(msg) conn = self.get_connection() return conn.elements.get_element_names( - dimension_name=self.dimension, hierarchy_name=self.hierarchy + dimension_name=dimension, hierarchy_name=hierarchy ) @add_viadot_metadata_columns - def to_df(self, if_empty: Literal["warn", "fail", "skip"] = "skip") -> pd.DataFrame: + def to_df( + self, + if_empty: Literal["warn", "fail", "skip"] = "skip", + mdx_query: str | None = None, + cube: str | None = None, + view: str | None = None, + ) -> pd.DataFrame: """Download data into a pandas DataFrame. To download the data to the dataframe user needs to specify MDX query - or combination of cube and view. + or combination of cube and view. Args: if_empty (Literal["warn", "fail", "skip"], optional): What to do if output - DataFrame is empty. Defaults to "skip". + DataFrame is empty. Defaults to "skip". + mdx_query (str): MDX query. Defaults to None. + cube (str): Cube name. Defaults to None. + view (str): View name. Defaults to None. Returns: pd.DataFrame: DataFrame with the data. Raises: ValidationError: When mdx and cube + view are not specified - or when combination of both is specified. + or when combination of both is specified. """ conn = self.get_connection() - if self.mdx_query is None and (self.cube is None or self.view is None): + if mdx_query is None and (cube is None or view is None): error_msg = "MDX query or cube and view are required." raise ValidationError(error_msg) - if self.mdx_query is not None and ( - self.cube is not None or self.view is not None - ): + if mdx_query is not None and (cube is not None or view is not None): error_msg = "Specify only one: MDX query or cube and view." raise ValidationError(error_msg) - if self.cube is not None and self.view is not None: + if cube is not None and view is not None: df = conn.cubes.cells.execute_view_dataframe( - cube_name=self.cube, - view_name=self.view, + cube_name=cube, + view_name=view, private=self.private, top=self.limit, ) - elif self.mdx_query is not None: - df = conn.cubes.cells.execute_mdx_dataframe(self.mdx_query) + elif mdx_query is not None: + df = conn.cubes.cells.execute_mdx_dataframe(mdx_query) self.logger.info( f"Data was successfully transformed into DataFrame: {len(df.columns)} columns and {len(df)} rows." ) - if df.empty is True: + if df.empty: self._handle_if_empty(if_empty) return df diff --git a/tests/unit/test_tm1.py b/tests/unit/test_tm1.py index 822eb34f8..ebbcb6edd 100644 --- a/tests/unit/test_tm1.py +++ b/tests/unit/test_tm1.py @@ -9,6 +9,10 @@ data = {"a": [420, 380, 390], "b": [50, 40, 45]} +test_cube = "test_cube" +test_view = "test_view" +test_dimension = "test_dim" +test_hierarchy = "test_hier" @pytest.fixture @@ -24,41 +28,17 @@ def tm1_credentials(): @pytest.fixture def tm1(tm1_credentials: TM1Credentials): return TM1( - credentials={ - "username": tm1_credentials.username, - "password": tm1_credentials.password, - "address": tm1_credentials.address, - "port": tm1_credentials.port, - }, + credentials=tm1_credentials.dict(), verify=False, - cube="test_cube", - view="test_view", - dimension="test_dim", - hierarchy="test_hier", ) -@pytest.fixture -def tm1_mock(tm1_credentials: TM1Credentials, mocker): - mocker.patch("viadot.sources.TM1.get_connection", return_value=True) - - return TM1( - credentials={ - "username": tm1_credentials.username, - "password": tm1_credentials.password, - "address": tm1_credentials.address, - "port": tm1_credentials.port, - }, - verify=False, - ) - - -def test_tm1_initialization(tm1_mock): +def test_tm1_initialization(tm1): """Test that the TM1 object is initialized with the correct credentials.""" - assert tm1_mock.credentials.get("address") == "localhost" - assert tm1_mock.credentials.get("username") == "test_user" + assert tm1.credentials.get("address") == "localhost" + assert tm1.credentials.get("username") == "test_user" assert ( - tm1_mock.credentials.get("password").get_secret_value() + tm1.credentials.get("password").get_secret_value() == "test_password" # pragma: allowlist secret ) @@ -75,7 +55,7 @@ def test_get_views_names(tm1): with patch("viadot.sources.tm1.TM1Service") as mock: instance = mock.return_value instance.views.get_all_names.return_value = ["view1", "view1"] - result = tm1.get_views_names() + result = tm1.get_views_names(cube=test_cube) assert result == ["view1", "view1"] @@ -91,7 +71,9 @@ def test_get_available_elements(tm1): with patch("viadot.sources.tm1.TM1Service") as mock: instance = mock.return_value instance.elements.get_element_names.return_value = ["el1", "el2"] - result = tm1.get_available_elements() + result = tm1.get_available_elements( + dimension=test_dimension, hierarchy=test_hierarchy + ) assert result == ["el1", "el2"] @@ -99,13 +81,14 @@ def test_to_df(tm1): with patch("viadot.sources.tm1.TM1Service") as mock: instance = mock.return_value instance.cubes.cells.execute_view_dataframe.return_value = pd.DataFrame(data) - result = tm1.to_df() + result = tm1.to_df(cube=test_cube, view=test_view) assert isinstance(result, pd.DataFrame) assert not result.empty assert len(result.columns) == 4 -def test_to_df_fail(tm1_mock): +def test_to_df_fail(tm1, mocker): + mocker.patch("viadot.sources.TM1.get_connection", return_value=True) with pytest.raises(ValidationError) as excinfo: - tm1_mock.to_df() + tm1.to_df() assert str(excinfo.value) == "MDX query or cube and view are required." From 54cfe72ae37b3da9348e965a3df86c644af61980 Mon Sep 17 00:00:00 2001 From: judynah Date: Wed, 4 Dec 2024 16:55:28 +0100 Subject: [PATCH 11/11] =?UTF-8?q?=E2=9E=95=20Added=20tm1py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 1 + requirements-dev.lock | 9 +++++++++ requirements.lock | 9 +++++++++ 3 files changed, 19 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 1544a7094..f9aa23a80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ dependencies = [ "paramiko>=3.5.0", # awswrangler 2.x. depends on pandas 1.x. "pandas<2.0", + "tm1py>=2.0.4", ] requires-python = ">=3.10" readme = "README.md" diff --git a/requirements-dev.lock b/requirements-dev.lock index 334283fd2..870eec83c 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -220,6 +220,8 @@ idna==3.7 # via httpx # via requests # via yarl +ijson==3.3.0 + # via tm1py imagehash==4.3.1 # via viadot2 importlib-metadata==6.11.0 @@ -315,6 +317,8 @@ mdit-py-plugins==0.4.1 # via jupytext mdurl==0.1.2 # via markdown-it-py +mdxpy==1.3.2 + # via tm1py mergedeep==1.3.4 # via mkdocs # via mkdocs-get-deps @@ -372,6 +376,7 @@ neoteroi-mkdocs==1.1.0 nest-asyncio==1.6.0 # via ipykernel networkx==3.3 + # via tm1py # via visions numpy==1.23.4 # via db-dtypes @@ -533,6 +538,7 @@ pytz==2024.1 # via dateparser # via pandas # via prefect + # via tm1py # via trino # via zeep pytzdata==2020.1 @@ -584,6 +590,7 @@ requests==2.32.3 # via responses # via sharepy # via simple-salesforce + # via tm1py # via trino # via viadot2 # via zeep @@ -675,6 +682,8 @@ tinycss2==1.3.0 # via cairosvg # via cssselect2 # via nbconvert +tm1py==2.0.4 + # via viadot2 toml==0.10.2 # via prefect tomli==2.0.1 diff --git a/requirements.lock b/requirements.lock index 1be587a19..dbc7b3184 100644 --- a/requirements.lock +++ b/requirements.lock @@ -162,6 +162,8 @@ idna==3.7 # via httpx # via requests # via yarl +ijson==3.3.0 + # via tm1py imagehash==4.3.1 # via viadot2 importlib-resources==6.1.3 @@ -200,6 +202,8 @@ markupsafe==2.1.5 # via mako mdurl==0.1.2 # via markdown-it-py +mdxpy==1.3.2 + # via tm1py more-itertools==10.5.0 # via simple-salesforce multidict==6.0.5 @@ -208,6 +212,7 @@ multidict==6.0.5 multimethod==1.12 # via visions networkx==3.3 + # via tm1py # via visions numpy==1.23.4 # via db-dtypes @@ -318,6 +323,7 @@ pytz==2024.1 # via dateparser # via pandas # via prefect + # via tm1py # via trino # via zeep pytzdata==2020.1 @@ -349,6 +355,7 @@ requests==2.32.3 # via requests-toolbelt # via sharepy # via simple-salesforce + # via tm1py # via trino # via viadot2 # via zeep @@ -420,6 +427,8 @@ tangled-up-in-unicode==0.2.0 # via visions text-unidecode==1.3 # via python-slugify +tm1py==2.0.4 + # via viadot2 toml==0.10.2 # via prefect trino==0.328.0