From 4527ce4cafcfadb3655f2ae6ecc2e0887bd23b37 Mon Sep 17 00:00:00 2001 From: Divyanshu Patel Date: Fri, 4 Oct 2024 19:00:09 +0530 Subject: [PATCH 01/19] external id --- soda/core/soda/common/aws_credentials.py | 2 +- soda/redshift/soda/data_sources/redshift_data_source.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/soda/core/soda/common/aws_credentials.py b/soda/core/soda/common/aws_credentials.py index 8dcac8ca9..36ca1957f 100644 --- a/soda/core/soda/common/aws_credentials.py +++ b/soda/core/soda/common/aws_credentials.py @@ -58,7 +58,7 @@ def assume_role(self, role_session_name: str): aws_session_token=self.session_token, ) - assumed_role_object = self.sts_client.assume_role(RoleArn=self.role_arn, RoleSessionName=role_session_name) + assumed_role_object = self.sts_client.assume_role(RoleArn=self.role_arn, ExternalId=self.external_id, RoleSessionName=role_session_name) credentials_dict = assumed_role_object["Credentials"] return AwsCredentials( region_name=self.region_name, diff --git a/soda/redshift/soda/data_sources/redshift_data_source.py b/soda/redshift/soda/data_sources/redshift_data_source.py index 9f07bacd3..a0c9b01b6 100644 --- a/soda/redshift/soda/data_sources/redshift_data_source.py +++ b/soda/redshift/soda/data_sources/redshift_data_source.py @@ -31,6 +31,7 @@ def __init__(self, logs: Logs, data_source_name: str, data_source_properties: di session_token=data_source_properties.get("session_token"), region_name=data_source_properties.get("region", "eu-west-1"), profile_name=data_source_properties.get("profile_name"), + external_id=data_source_properties.get("external_id"), ) self.username, self.password = self.__get_cluster_credentials(aws_credentials) @@ -52,9 +53,11 @@ def __get_cluster_credentials(self, aws_credentials: AwsCredentials): role_session_name="soda_redshift_get_cluster_credentials" ) + region_name = resolved_aws_credentials.region_name or "eu-west-1" + client = boto3.client( "redshift", - region_name=resolved_aws_credentials.region_name, + region_name=region_name, aws_access_key_id=resolved_aws_credentials.access_key_id, aws_secret_access_key=resolved_aws_credentials.secret_access_key, aws_session_token=resolved_aws_credentials.session_token, From aa4f11450f0600dbd6a5aa5ff86dfef720fc3c4c Mon Sep 17 00:00:00 2001 From: Divyanshu Patel Date: Fri, 4 Oct 2024 20:30:47 +0530 Subject: [PATCH 02/19] fix? --- soda/redshift/soda/data_sources/redshift_data_source.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/soda/redshift/soda/data_sources/redshift_data_source.py b/soda/redshift/soda/data_sources/redshift_data_source.py index a0c9b01b6..f37c42e6b 100644 --- a/soda/redshift/soda/data_sources/redshift_data_source.py +++ b/soda/redshift/soda/data_sources/redshift_data_source.py @@ -64,8 +64,9 @@ def __get_cluster_credentials(self, aws_credentials: AwsCredentials): ) cluster_name = self.host.split(".")[0] - username = self.username - db_name = self.database + username = self.username or self.data_source_properties.get("username") + db_name = self.data_source_properties.get("database") + cluster_creds = client.get_cluster_credentials( DbUser=username, DbName=db_name, ClusterIdentifier=cluster_name, AutoCreate=False, DurationSeconds=3600 ) From 7bbb6b42e486eb54cbb7131c4ea147e5f036c78c Mon Sep 17 00:00:00 2001 From: Divyanshu Patel Date: Fri, 4 Oct 2024 21:47:58 +0530 Subject: [PATCH 03/19] send db user and db name --- soda/redshift/soda/data_sources/redshift_data_source.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/soda/redshift/soda/data_sources/redshift_data_source.py b/soda/redshift/soda/data_sources/redshift_data_source.py index f37c42e6b..2db6f69a0 100644 --- a/soda/redshift/soda/data_sources/redshift_data_source.py +++ b/soda/redshift/soda/data_sources/redshift_data_source.py @@ -31,6 +31,8 @@ def __init__(self, logs: Logs, data_source_name: str, data_source_properties: di session_token=data_source_properties.get("session_token"), region_name=data_source_properties.get("region", "eu-west-1"), profile_name=data_source_properties.get("profile_name"), + username=data_source_properties.get("dbuser"), + db_name=data_source_properties.get("dbname"), external_id=data_source_properties.get("external_id"), ) self.username, self.password = self.__get_cluster_credentials(aws_credentials) From 3b0afe861b57b42a14e5f8388cc6aee34d5a9f4d Mon Sep 17 00:00:00 2001 From: Divyanshu Patel Date: Fri, 4 Oct 2024 21:50:55 +0530 Subject: [PATCH 04/19] db_name --- soda/redshift/soda/data_sources/redshift_data_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/soda/redshift/soda/data_sources/redshift_data_source.py b/soda/redshift/soda/data_sources/redshift_data_source.py index 2db6f69a0..b5044f5a3 100644 --- a/soda/redshift/soda/data_sources/redshift_data_source.py +++ b/soda/redshift/soda/data_sources/redshift_data_source.py @@ -67,7 +67,7 @@ def __get_cluster_credentials(self, aws_credentials: AwsCredentials): cluster_name = self.host.split(".")[0] username = self.username or self.data_source_properties.get("username") - db_name = self.data_source_properties.get("database") + db_name = self.data_source_properties.get("db_name") cluster_creds = client.get_cluster_credentials( DbUser=username, DbName=db_name, ClusterIdentifier=cluster_name, AutoCreate=False, DurationSeconds=3600 From f319329142269497b64455dc359a73d9aaf52d82 Mon Sep 17 00:00:00 2001 From: Divyanshu Patel Date: Fri, 4 Oct 2024 22:27:12 +0530 Subject: [PATCH 05/19] Revert "db_name" This reverts commit 3b0afe861b57b42a14e5f8388cc6aee34d5a9f4d. --- soda/redshift/soda/data_sources/redshift_data_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/soda/redshift/soda/data_sources/redshift_data_source.py b/soda/redshift/soda/data_sources/redshift_data_source.py index b5044f5a3..2db6f69a0 100644 --- a/soda/redshift/soda/data_sources/redshift_data_source.py +++ b/soda/redshift/soda/data_sources/redshift_data_source.py @@ -67,7 +67,7 @@ def __get_cluster_credentials(self, aws_credentials: AwsCredentials): cluster_name = self.host.split(".")[0] username = self.username or self.data_source_properties.get("username") - db_name = self.data_source_properties.get("db_name") + db_name = self.data_source_properties.get("database") cluster_creds = client.get_cluster_credentials( DbUser=username, DbName=db_name, ClusterIdentifier=cluster_name, AutoCreate=False, DurationSeconds=3600 From 28781764003fc5cdd918fc03a58727972f2a2513 Mon Sep 17 00:00:00 2001 From: Divyanshu Patel Date: Fri, 4 Oct 2024 22:27:23 +0530 Subject: [PATCH 06/19] Revert "send db user and db name" This reverts commit 7bbb6b42e486eb54cbb7131c4ea147e5f036c78c. --- soda/redshift/soda/data_sources/redshift_data_source.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/soda/redshift/soda/data_sources/redshift_data_source.py b/soda/redshift/soda/data_sources/redshift_data_source.py index 2db6f69a0..f37c42e6b 100644 --- a/soda/redshift/soda/data_sources/redshift_data_source.py +++ b/soda/redshift/soda/data_sources/redshift_data_source.py @@ -31,8 +31,6 @@ def __init__(self, logs: Logs, data_source_name: str, data_source_properties: di session_token=data_source_properties.get("session_token"), region_name=data_source_properties.get("region", "eu-west-1"), profile_name=data_source_properties.get("profile_name"), - username=data_source_properties.get("dbuser"), - db_name=data_source_properties.get("dbname"), external_id=data_source_properties.get("external_id"), ) self.username, self.password = self.__get_cluster_credentials(aws_credentials) From f5288f4651b1b937493c8b0b0cceee219ea76fb1 Mon Sep 17 00:00:00 2001 From: Divyanshu Patel Date: Fri, 4 Oct 2024 22:27:37 +0530 Subject: [PATCH 07/19] Revert "fix?" This reverts commit aa4f11450f0600dbd6a5aa5ff86dfef720fc3c4c. --- soda/redshift/soda/data_sources/redshift_data_source.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/soda/redshift/soda/data_sources/redshift_data_source.py b/soda/redshift/soda/data_sources/redshift_data_source.py index f37c42e6b..a0c9b01b6 100644 --- a/soda/redshift/soda/data_sources/redshift_data_source.py +++ b/soda/redshift/soda/data_sources/redshift_data_source.py @@ -64,9 +64,8 @@ def __get_cluster_credentials(self, aws_credentials: AwsCredentials): ) cluster_name = self.host.split(".")[0] - username = self.username or self.data_source_properties.get("username") - db_name = self.data_source_properties.get("database") - + username = self.username + db_name = self.database cluster_creds = client.get_cluster_credentials( DbUser=username, DbName=db_name, ClusterIdentifier=cluster_name, AutoCreate=False, DurationSeconds=3600 ) From 07bb44bfc931a42119fa8a4876c7426aeb373666 Mon Sep 17 00:00:00 2001 From: Divyanshu Patel Date: Fri, 4 Oct 2024 23:03:25 +0530 Subject: [PATCH 08/19] changes --- soda/redshift/soda/data_sources/redshift_data_source.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/soda/redshift/soda/data_sources/redshift_data_source.py b/soda/redshift/soda/data_sources/redshift_data_source.py index a0c9b01b6..a4fd7cb56 100644 --- a/soda/redshift/soda/data_sources/redshift_data_source.py +++ b/soda/redshift/soda/data_sources/redshift_data_source.py @@ -22,6 +22,8 @@ def __init__(self, logs: Logs, data_source_name: str, data_source_properties: di self.connect_timeout = data_source_properties.get("connection_timeout_sec") self.username = data_source_properties.get("username") self.password = data_source_properties.get("password") + self.dbuser = data_source_properties.get("dbuser") + self.dbname = data_source_properties.get("dbname") if not self.username or not self.password: aws_credentials = AwsCredentials( @@ -64,8 +66,8 @@ def __get_cluster_credentials(self, aws_credentials: AwsCredentials): ) cluster_name = self.host.split(".")[0] - username = self.username - db_name = self.database + username = self.dbuser if self.dbuser else self.username + db_name = self.dbname if self.dbname else self.database cluster_creds = client.get_cluster_credentials( DbUser=username, DbName=db_name, ClusterIdentifier=cluster_name, AutoCreate=False, DurationSeconds=3600 ) From 3ff0fd3adcc1fa390baeb9fd3c143d4b8c9edf88 Mon Sep 17 00:00:00 2001 From: Divyanshu Patel Date: Thu, 10 Oct 2024 22:31:59 +0530 Subject: [PATCH 09/19] update default region --- soda/redshift/soda/data_sources/redshift_data_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/soda/redshift/soda/data_sources/redshift_data_source.py b/soda/redshift/soda/data_sources/redshift_data_source.py index a4fd7cb56..4d8bde9f0 100644 --- a/soda/redshift/soda/data_sources/redshift_data_source.py +++ b/soda/redshift/soda/data_sources/redshift_data_source.py @@ -55,7 +55,7 @@ def __get_cluster_credentials(self, aws_credentials: AwsCredentials): role_session_name="soda_redshift_get_cluster_credentials" ) - region_name = resolved_aws_credentials.region_name or "eu-west-1" + region_name = resolved_aws_credentials.region_name or "us-west-2" client = boto3.client( "redshift", From 6086d6acfbfc246ffccf4a2892b6502a3bf502c5 Mon Sep 17 00:00:00 2001 From: Divyanshu Patel Date: Thu, 10 Oct 2024 22:57:35 +0530 Subject: [PATCH 10/19] with iam --- soda/redshift/soda/data_sources/redshift_data_source.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/soda/redshift/soda/data_sources/redshift_data_source.py b/soda/redshift/soda/data_sources/redshift_data_source.py index 4d8bde9f0..685ab27fb 100644 --- a/soda/redshift/soda/data_sources/redshift_data_source.py +++ b/soda/redshift/soda/data_sources/redshift_data_source.py @@ -68,9 +68,12 @@ def __get_cluster_credentials(self, aws_credentials: AwsCredentials): cluster_name = self.host.split(".")[0] username = self.dbuser if self.dbuser else self.username db_name = self.dbname if self.dbname else self.database - cluster_creds = client.get_cluster_credentials( - DbUser=username, DbName=db_name, ClusterIdentifier=cluster_name, AutoCreate=False, DurationSeconds=3600 + cluster_creds = client.get_cluster_credentials_with_iam( + DbName=db_name, ClusterIdentifier=cluster_name, AutoCreate=False, DurationSeconds=3600 ) + # cluster_creds = client.get_cluster_credentials( + # DbUser=username, DbName=db_name, ClusterIdentifier=cluster_name, AutoCreate=False, DurationSeconds=3600 + # ) return cluster_creds["DbUser"], cluster_creds["DbPassword"] From a9af3a1e805cfa2099c764504c555d755cc08d6d Mon Sep 17 00:00:00 2001 From: Divyanshu Patel Date: Thu, 10 Oct 2024 23:24:34 +0530 Subject: [PATCH 11/19] cluster iam --- soda/redshift/soda/data_sources/redshift_data_source.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/soda/redshift/soda/data_sources/redshift_data_source.py b/soda/redshift/soda/data_sources/redshift_data_source.py index 685ab27fb..3e760f179 100644 --- a/soda/redshift/soda/data_sources/redshift_data_source.py +++ b/soda/redshift/soda/data_sources/redshift_data_source.py @@ -68,12 +68,10 @@ def __get_cluster_credentials(self, aws_credentials: AwsCredentials): cluster_name = self.host.split(".")[0] username = self.dbuser if self.dbuser else self.username db_name = self.dbname if self.dbname else self.database + cluster_creds = client.get_cluster_credentials_with_iam( - DbName=db_name, ClusterIdentifier=cluster_name, AutoCreate=False, DurationSeconds=3600 + DbName=db_name, ClusterIdentifier=cluster_name, DurationSeconds=3600 ) - # cluster_creds = client.get_cluster_credentials( - # DbUser=username, DbName=db_name, ClusterIdentifier=cluster_name, AutoCreate=False, DurationSeconds=3600 - # ) return cluster_creds["DbUser"], cluster_creds["DbPassword"] From 83fc001fe63dcb8ffc4994f82181c96b352032df Mon Sep 17 00:00:00 2001 From: Divyanshu Patel Date: Thu, 10 Oct 2024 23:55:20 +0530 Subject: [PATCH 12/19] Revert "cluster iam" This reverts commit a9af3a1e805cfa2099c764504c555d755cc08d6d. --- soda/redshift/soda/data_sources/redshift_data_source.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/soda/redshift/soda/data_sources/redshift_data_source.py b/soda/redshift/soda/data_sources/redshift_data_source.py index 3e760f179..685ab27fb 100644 --- a/soda/redshift/soda/data_sources/redshift_data_source.py +++ b/soda/redshift/soda/data_sources/redshift_data_source.py @@ -68,10 +68,12 @@ def __get_cluster_credentials(self, aws_credentials: AwsCredentials): cluster_name = self.host.split(".")[0] username = self.dbuser if self.dbuser else self.username db_name = self.dbname if self.dbname else self.database - cluster_creds = client.get_cluster_credentials_with_iam( - DbName=db_name, ClusterIdentifier=cluster_name, DurationSeconds=3600 + DbName=db_name, ClusterIdentifier=cluster_name, AutoCreate=False, DurationSeconds=3600 ) + # cluster_creds = client.get_cluster_credentials( + # DbUser=username, DbName=db_name, ClusterIdentifier=cluster_name, AutoCreate=False, DurationSeconds=3600 + # ) return cluster_creds["DbUser"], cluster_creds["DbPassword"] From a3353bd4da4f05f174f046e59e5365928aeed3b2 Mon Sep 17 00:00:00 2001 From: Divyanshu Patel Date: Thu, 10 Oct 2024 23:55:34 +0530 Subject: [PATCH 13/19] Revert "with iam" This reverts commit 6086d6acfbfc246ffccf4a2892b6502a3bf502c5. --- soda/redshift/soda/data_sources/redshift_data_source.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/soda/redshift/soda/data_sources/redshift_data_source.py b/soda/redshift/soda/data_sources/redshift_data_source.py index 685ab27fb..4d8bde9f0 100644 --- a/soda/redshift/soda/data_sources/redshift_data_source.py +++ b/soda/redshift/soda/data_sources/redshift_data_source.py @@ -68,12 +68,9 @@ def __get_cluster_credentials(self, aws_credentials: AwsCredentials): cluster_name = self.host.split(".")[0] username = self.dbuser if self.dbuser else self.username db_name = self.dbname if self.dbname else self.database - cluster_creds = client.get_cluster_credentials_with_iam( - DbName=db_name, ClusterIdentifier=cluster_name, AutoCreate=False, DurationSeconds=3600 + cluster_creds = client.get_cluster_credentials( + DbUser=username, DbName=db_name, ClusterIdentifier=cluster_name, AutoCreate=False, DurationSeconds=3600 ) - # cluster_creds = client.get_cluster_credentials( - # DbUser=username, DbName=db_name, ClusterIdentifier=cluster_name, AutoCreate=False, DurationSeconds=3600 - # ) return cluster_creds["DbUser"], cluster_creds["DbPassword"] From 572f89d0d5ab4ca5772f53e4b41da53d70766a63 Mon Sep 17 00:00:00 2001 From: Divyanshu Patel Date: Fri, 11 Oct 2024 00:00:02 +0530 Subject: [PATCH 14/19] cluster name --- soda/redshift/soda/data_sources/redshift_data_source.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/soda/redshift/soda/data_sources/redshift_data_source.py b/soda/redshift/soda/data_sources/redshift_data_source.py index 4d8bde9f0..10fd3f584 100644 --- a/soda/redshift/soda/data_sources/redshift_data_source.py +++ b/soda/redshift/soda/data_sources/redshift_data_source.py @@ -24,6 +24,7 @@ def __init__(self, logs: Logs, data_source_name: str, data_source_properties: di self.password = data_source_properties.get("password") self.dbuser = data_source_properties.get("dbuser") self.dbname = data_source_properties.get("dbname") + self.cluster_id = data_source_properties.get("cluster_id") if not self.username or not self.password: aws_credentials = AwsCredentials( @@ -65,7 +66,7 @@ def __get_cluster_credentials(self, aws_credentials: AwsCredentials): aws_session_token=resolved_aws_credentials.session_token, ) - cluster_name = self.host.split(".")[0] + cluster_name = self.cluster_id username = self.dbuser if self.dbuser else self.username db_name = self.dbname if self.dbname else self.database cluster_creds = client.get_cluster_credentials( From 3824b3cae4431bb1097d94dfade0575411170c15 Mon Sep 17 00:00:00 2001 From: Divyanshu Patel Date: Fri, 11 Oct 2024 01:18:26 +0530 Subject: [PATCH 15/19] hardcode --- soda/redshift/soda/data_sources/redshift_data_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/soda/redshift/soda/data_sources/redshift_data_source.py b/soda/redshift/soda/data_sources/redshift_data_source.py index 10fd3f584..e49676ca5 100644 --- a/soda/redshift/soda/data_sources/redshift_data_source.py +++ b/soda/redshift/soda/data_sources/redshift_data_source.py @@ -66,7 +66,7 @@ def __get_cluster_credentials(self, aws_credentials: AwsCredentials): aws_session_token=resolved_aws_credentials.session_token, ) - cluster_name = self.cluster_id + cluster_name = "bi-edw-db" username = self.dbuser if self.dbuser else self.username db_name = self.dbname if self.dbname else self.database cluster_creds = client.get_cluster_credentials( From 36d5763196d8b4900c50c3d9ac8c4f5e610a1c35 Mon Sep 17 00:00:00 2001 From: Divyanshu Patel Date: Fri, 11 Oct 2024 02:06:13 +0530 Subject: [PATCH 16/19] logs --- .../soda/data_sources/redshift_data_source.py | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/soda/redshift/soda/data_sources/redshift_data_source.py b/soda/redshift/soda/data_sources/redshift_data_source.py index e49676ca5..d2e793b57 100644 --- a/soda/redshift/soda/data_sources/redshift_data_source.py +++ b/soda/redshift/soda/data_sources/redshift_data_source.py @@ -10,6 +10,15 @@ logger = logging.getLogger(__name__) +logging.basicConfig( + level=logging.DEBUG, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + handlers=[ + logging.StreamHandler(), + ] +) + +logger = logging.getLogger(__name__) class RedshiftDataSource(DataSource): TYPE = "redshift" @@ -55,8 +64,10 @@ def __get_cluster_credentials(self, aws_credentials: AwsCredentials): resolved_aws_credentials = aws_credentials.resolve_role( role_session_name="soda_redshift_get_cluster_credentials" ) + logger.debug(f"Resolved AWS Credentials: {resolved_aws_credentials}") region_name = resolved_aws_credentials.region_name or "us-west-2" + logger.debug(f"Region Name: {region_name}") client = boto3.client( "redshift", @@ -65,13 +76,28 @@ def __get_cluster_credentials(self, aws_credentials: AwsCredentials): aws_secret_access_key=resolved_aws_credentials.secret_access_key, aws_session_token=resolved_aws_credentials.session_token, ) + logger.debug("Boto3 Redshift client initialized.") cluster_name = "bi-edw-db" username = self.dbuser if self.dbuser else self.username db_name = self.dbname if self.dbname else self.database - cluster_creds = client.get_cluster_credentials( - DbUser=username, DbName=db_name, ClusterIdentifier=cluster_name, AutoCreate=False, DurationSeconds=3600 - ) + + logger.debug(f"Using cluster name: {cluster_name}") + logger.debug(f"Using DbUser: {username}") + logger.debug(f"Using DbName: {db_name}") + + try: + cluster_creds = client.get_cluster_credentials( + DbUser=username, + DbName=db_name, + ClusterIdentifier=cluster_name, + AutoCreate=False, + DurationSeconds=3600 + ) + logger.debug(f"Cluster Credentials Retrieved: {cluster_creds}") + except botocore.exceptions.ClientError as error: + logger.error(f"Error retrieving cluster credentials: {error}") + raise return cluster_creds["DbUser"], cluster_creds["DbPassword"] From cd47e3351c0fcfa968ab6bd3ff4c6fb6dbffdc65 Mon Sep 17 00:00:00 2001 From: Divyanshu Patel Date: Fri, 11 Oct 2024 02:47:14 +0530 Subject: [PATCH 17/19] Revert "logs" This reverts commit 36d5763196d8b4900c50c3d9ac8c4f5e610a1c35. --- .../soda/data_sources/redshift_data_source.py | 32 ++----------------- 1 file changed, 3 insertions(+), 29 deletions(-) diff --git a/soda/redshift/soda/data_sources/redshift_data_source.py b/soda/redshift/soda/data_sources/redshift_data_source.py index d2e793b57..e49676ca5 100644 --- a/soda/redshift/soda/data_sources/redshift_data_source.py +++ b/soda/redshift/soda/data_sources/redshift_data_source.py @@ -10,15 +10,6 @@ logger = logging.getLogger(__name__) -logging.basicConfig( - level=logging.DEBUG, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - handlers=[ - logging.StreamHandler(), - ] -) - -logger = logging.getLogger(__name__) class RedshiftDataSource(DataSource): TYPE = "redshift" @@ -64,10 +55,8 @@ def __get_cluster_credentials(self, aws_credentials: AwsCredentials): resolved_aws_credentials = aws_credentials.resolve_role( role_session_name="soda_redshift_get_cluster_credentials" ) - logger.debug(f"Resolved AWS Credentials: {resolved_aws_credentials}") region_name = resolved_aws_credentials.region_name or "us-west-2" - logger.debug(f"Region Name: {region_name}") client = boto3.client( "redshift", @@ -76,28 +65,13 @@ def __get_cluster_credentials(self, aws_credentials: AwsCredentials): aws_secret_access_key=resolved_aws_credentials.secret_access_key, aws_session_token=resolved_aws_credentials.session_token, ) - logger.debug("Boto3 Redshift client initialized.") cluster_name = "bi-edw-db" username = self.dbuser if self.dbuser else self.username db_name = self.dbname if self.dbname else self.database - - logger.debug(f"Using cluster name: {cluster_name}") - logger.debug(f"Using DbUser: {username}") - logger.debug(f"Using DbName: {db_name}") - - try: - cluster_creds = client.get_cluster_credentials( - DbUser=username, - DbName=db_name, - ClusterIdentifier=cluster_name, - AutoCreate=False, - DurationSeconds=3600 - ) - logger.debug(f"Cluster Credentials Retrieved: {cluster_creds}") - except botocore.exceptions.ClientError as error: - logger.error(f"Error retrieving cluster credentials: {error}") - raise + cluster_creds = client.get_cluster_credentials( + DbUser=username, DbName=db_name, ClusterIdentifier=cluster_name, AutoCreate=False, DurationSeconds=3600 + ) return cluster_creds["DbUser"], cluster_creds["DbPassword"] From 62c86671452f74e719f0938632311354de61b850 Mon Sep 17 00:00:00 2001 From: Divyanshu Patel Date: Fri, 11 Oct 2024 02:56:34 +0530 Subject: [PATCH 18/19] hardcode --- soda/redshift/soda/data_sources/redshift_data_source.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/soda/redshift/soda/data_sources/redshift_data_source.py b/soda/redshift/soda/data_sources/redshift_data_source.py index e49676ca5..95b67f3fb 100644 --- a/soda/redshift/soda/data_sources/redshift_data_source.py +++ b/soda/redshift/soda/data_sources/redshift_data_source.py @@ -67,8 +67,8 @@ def __get_cluster_credentials(self, aws_credentials: AwsCredentials): ) cluster_name = "bi-edw-db" - username = self.dbuser if self.dbuser else self.username - db_name = self.dbname if self.dbname else self.database + username = "atlan_user" + db_name = "bi_edw_db" cluster_creds = client.get_cluster_credentials( DbUser=username, DbName=db_name, ClusterIdentifier=cluster_name, AutoCreate=False, DurationSeconds=3600 ) From e4dc0937bea26f14cfc57e9188c6a84da602b653 Mon Sep 17 00:00:00 2001 From: Divyanshu Patel Date: Fri, 11 Oct 2024 03:58:51 +0530 Subject: [PATCH 19/19] hardcode get_cluster_credentials --- soda/redshift/soda/data_sources/redshift_data_source.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/soda/redshift/soda/data_sources/redshift_data_source.py b/soda/redshift/soda/data_sources/redshift_data_source.py index 95b67f3fb..fc310993a 100644 --- a/soda/redshift/soda/data_sources/redshift_data_source.py +++ b/soda/redshift/soda/data_sources/redshift_data_source.py @@ -66,11 +66,12 @@ def __get_cluster_credentials(self, aws_credentials: AwsCredentials): aws_session_token=resolved_aws_credentials.session_token, ) - cluster_name = "bi-edw-db" - username = "atlan_user" - db_name = "bi_edw_db" cluster_creds = client.get_cluster_credentials( - DbUser=username, DbName=db_name, ClusterIdentifier=cluster_name, AutoCreate=False, DurationSeconds=3600 + DbUser="atlan_user", + DbName="bi_edw_db", + ClusterIdentifier="bi-edw-db", + AutoCreate=False, + DurationSeconds=3600 ) return cluster_creds["DbUser"], cluster_creds["DbPassword"]