From c4d53bba8c03a0eb9de60f18a4d343bf6a40dcc2 Mon Sep 17 00:00:00 2001 From: EvanBrown96 Date: Fri, 17 Mar 2023 09:09:51 -0400 Subject: [PATCH 1/3] add caching for ssm parameters and datadog values pulled when creating sessions --- almdrlib/environment.py | 78 ++++++++++++++++++++++++++------------- tests/test_environment.py | 5 +-- 2 files changed, 54 insertions(+), 29 deletions(-) diff --git a/almdrlib/environment.py b/almdrlib/environment.py index 6ee80cf..9d60587 100644 --- a/almdrlib/environment.py +++ b/almdrlib/environment.py @@ -46,6 +46,7 @@ import boto3 import os import botocore +import cachetools class AlEnvException(Exception): @@ -64,32 +65,22 @@ class AlmdrlibSourceNotEnabledError(AlEnvException): class AlEnv: + + dynamodb = None + ssm = None + table = None + def __init__(self, application_name, client=None, source=("dynamodb",)): self.application_name = application_name self.client = client self.source = source if type(source) in [tuple, list] else (source,) - self.region = AlEnv._get_region() - self.stack_name = AlEnv._get_stack_name() - self.table_name = AlEnv._table_name(self.region, self.stack_name) - try: - self.dynamodb = boto3.resource('dynamodb') - self.ssm = boto3.client('ssm') - except (botocore.exceptions.NoRegionError, botocore.exceptions.NoCredentialsError) as e: - raise AlEnvAwsConfigurationException(f'Please validate your AWS configuration') from e - if "dynamodb" in source: - try: - self.table = self.dynamodb.Table(self.table_name) - self._table_date_time = self.table.creation_date_time - except botocore.exceptions.ClientError as e: - if e.response['Error']['Code'] == 'ResourceNotFoundException': - raise AlEnvConfigurationTableUnavailableException(self.table_name) - else: - raise AlEnvException() from e + + self._setup_aws_resources(source) def get(self, key, default=None, format='decoded', type=None): if "dynamodb" not in self.source: raise AlmdrlibSourceNotEnabledError("dynamodb is not enabled for this environment") - fetched_value = self.table.get_item(Key={"key": self._make_ddb_key(key)}).get('Item', {}).get('value') + fetched_value = AlEnv._get_cached_dynamodb_item(self._make_ddb_key(key)) converted = AlEnv._convert(fetched_value, format, type) if converted is not None: return converted @@ -99,16 +90,10 @@ def get(self, key, default=None, format='decoded', type=None): def get_parameter(self, key, default=None, decrypt=False): if "ssm" not in self.source: raise AlmdrlibSourceNotEnabledError("ssm is not enabled for this environment") - try: - parameter = self.ssm.get_parameter(Name=self._make_ssm_key(key), WithDecryption=decrypt) - except self.ssm.exceptions.ParameterNotFound: - return default - except botocore.exceptions.ClientError as e: - raise AlEnvException() from e - return parameter["Parameter"]["Value"] + return AlEnv._get_cached_ssm_parameter(self._make_ssm_key(key), default, decrypt) def _make_ssm_key(self, option_key): - return f"/deployments/{self.stack_name}/{self._get_region()}/env-settings/{self.application_name}/{self._make_client_option_key(option_key)}" + return f"/deployments/{self._get_stack_name()}/{self._get_region()}/env-settings/{self.application_name}/{self._make_client_option_key(option_key)}" def _make_ddb_key(self, option_key): return f"{self.application_name}.{self._make_client_option_key(option_key)}" @@ -118,7 +103,48 @@ def _make_client_option_key(self, option_key): return option_key else: return f"{self.client}.{option_key}" + + @staticmethod + @cachetools.cached(cache=cachetools.TTLCache(maxsize=16, ttl=3600)) + def _get_cached_ssm_parameter(ssm_key, default=None, decrypt=False): + try: + parameter = AlEnv.ssm.get_parameter(Name=ssm_key, WithDecryption=decrypt) + except AlEnv.ssm.exceptions.ParameterNotFound: + return default + except botocore.exceptions.ClientError as e: + raise AlEnvException() from e + return parameter["Parameter"]["Value"] + + @staticmethod + @cachetools.cached(cache=cachetools.TTLCache(maxsize=16, ttl=3600)) + def _get_cached_dynamodb_item(key): + return AlEnv.table.get_item(Key={"key": key}).get('Item', {}).get('value') + + @staticmethod + def _setup_aws_resources(source): + try: + if "dynamodb" in source: + if AlEnv.dynamodb is None: + AlEnv.dynamodb = boto3.resource('dynamodb') + if AlEnv.table is None: + AlEnv._setup_dynamodb_table() + AlEnv.ssm = boto3.client('ssm') + except (botocore.exceptions.NoRegionError, botocore.exceptions.NoCredentialsError) as e: + raise AlEnvAwsConfigurationException(f'Please validate your AWS configuration') from e + @staticmethod + def _setup_dynamodb_table(): + region = AlEnv._get_region() + stack_name = AlEnv._get_stack_name() + table_name = AlEnv._table_name(region, stack_name) + try: + AlEnv.table = AlEnv.dynamodb.Table(table_name) + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == 'ResourceNotFoundException': + raise AlEnvConfigurationTableUnavailableException(table_name) + else: + raise AlEnvException() from e + @staticmethod def _convert(value, format, type): if format == 'raw' and value: diff --git a/tests/test_environment.py b/tests/test_environment.py index 2c741c2..3fbe94b 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -8,7 +8,7 @@ class MockDDBTable: creation_date_time = '2005-08-09T18:31:42-03:30' def __init__(self, tablename): - pass + self.tablename = tablename def get_item(self, **kwargs): Key = kwargs.get('Key')['key'] @@ -61,7 +61,6 @@ def get_parameter(**kwargs): else: raise MockBotoSSM.exceptions.ParameterNotFound() - class TestAlEnv(unittest.TestCase): def test_something(self): boto3.resource = MagicMock(return_value=MockBotoDDB) @@ -69,7 +68,7 @@ def test_something(self): os.environ['ALERTLOGIC_STACK_REGION'] = 'us-west-1' os.environ['ALERTLOGIC_STACK_NAME'] = 'production' env = AlEnv("someapplication", source="dynamodb") - assert env.table_name == 'us-west-1.production.dev.global.settings' + assert AlEnv.table.tablename == 'us-west-1.production.dev.global.settings' assert env.get("strkey") == 'strvalue' assert env.get("strkey", format='raw') == '"strvalue"' assert env.get("intkey") == '1' From 3a82ffe0271fbcc9d2df8f81c00068a4ed96a21c Mon Sep 17 00:00:00 2001 From: EvanBrown96 Date: Fri, 17 Mar 2023 09:41:18 -0400 Subject: [PATCH 2/3] add some logging --- almdrlib/environment.py | 15 ++++++++++----- tests/test_environment.py | 4 ++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/almdrlib/environment.py b/almdrlib/environment.py index 9d60587..ee148a6 100644 --- a/almdrlib/environment.py +++ b/almdrlib/environment.py @@ -47,16 +47,18 @@ import os import botocore import cachetools +import logging + + +logger = logging.getLogger(__name__) class AlEnvException(Exception): pass - class AlEnvAwsConfigurationException(AlEnvException): pass - class AlEnvConfigurationTableUnavailableException(AlEnvException): pass @@ -69,6 +71,7 @@ class AlEnv: dynamodb = None ssm = None table = None + table_name = None def __init__(self, application_name, client=None, source=("dynamodb",)): self.application_name = application_name @@ -108,6 +111,7 @@ def _make_client_option_key(self, option_key): @cachetools.cached(cache=cachetools.TTLCache(maxsize=16, ttl=3600)) def _get_cached_ssm_parameter(ssm_key, default=None, decrypt=False): try: + logger.debug(f"Query SSM for parameter {ssm_key}") parameter = AlEnv.ssm.get_parameter(Name=ssm_key, WithDecryption=decrypt) except AlEnv.ssm.exceptions.ParameterNotFound: return default @@ -118,6 +122,7 @@ def _get_cached_ssm_parameter(ssm_key, default=None, decrypt=False): @staticmethod @cachetools.cached(cache=cachetools.TTLCache(maxsize=16, ttl=3600)) def _get_cached_dynamodb_item(key): + logger.debug(f"Query DynamoDB table {AlEnv.table_name} for item {key}") return AlEnv.table.get_item(Key={"key": key}).get('Item', {}).get('value') @staticmethod @@ -136,12 +141,12 @@ def _setup_aws_resources(source): def _setup_dynamodb_table(): region = AlEnv._get_region() stack_name = AlEnv._get_stack_name() - table_name = AlEnv._table_name(region, stack_name) + AlEnv.table_name = AlEnv._table_name(region, stack_name) try: - AlEnv.table = AlEnv.dynamodb.Table(table_name) + AlEnv.table = AlEnv.dynamodb.Table(AlEnv.table_name) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == 'ResourceNotFoundException': - raise AlEnvConfigurationTableUnavailableException(table_name) + raise AlEnvConfigurationTableUnavailableException(AlEnv.table_name) else: raise AlEnvException() from e diff --git a/tests/test_environment.py b/tests/test_environment.py index 3fbe94b..bffddc4 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -8,7 +8,7 @@ class MockDDBTable: creation_date_time = '2005-08-09T18:31:42-03:30' def __init__(self, tablename): - self.tablename = tablename + pass def get_item(self, **kwargs): Key = kwargs.get('Key')['key'] @@ -68,7 +68,7 @@ def test_something(self): os.environ['ALERTLOGIC_STACK_REGION'] = 'us-west-1' os.environ['ALERTLOGIC_STACK_NAME'] = 'production' env = AlEnv("someapplication", source="dynamodb") - assert AlEnv.table.tablename == 'us-west-1.production.dev.global.settings' + assert AlEnv.table_name == 'us-west-1.production.dev.global.settings' assert env.get("strkey") == 'strvalue' assert env.get("strkey", format='raw') == '"strvalue"' assert env.get("intkey") == '1' From 6baa3bb9adce5a41c33b133fdef5dc9cb771d1af Mon Sep 17 00:00:00 2001 From: EvanBrown96 Date: Tue, 2 May 2023 16:15:37 -0400 Subject: [PATCH 3/3] add cachetools to requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index db4e069..c19d561 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ pyyaml==5.4.1 jsonschema[format_nongpl]==3.2.0 m2r2==0.3.2 boto3>=1.16.57 +cachetools>=5.3.0 \ No newline at end of file