diff --git a/almdrlib/config.py b/almdrlib/config.py index 28ef05c..61e583b 100644 --- a/almdrlib/config.py +++ b/almdrlib/config.py @@ -6,6 +6,7 @@ import json import almdrlib.constants from almdrlib.exceptions import AlmdrlibValueError +from almdrlib.environment import AlEnv logger = logging.getLogger(__name__) @@ -32,6 +33,7 @@ class Config(): ALERTLOGIC_ACCOUNT_ID - Account Id to perform operations against. ALERTLOGIC_RESIDENCY - Data Residency when creating new deployments ALERTLOGIC_API - Directory where OpenAPI yaml files reside + ALERTLOGIC_SERVICE_NAME - If a (micro)service is built around almdrlib used as identifier Config File section values access_key_id - User's AIMS Access Key ID @@ -51,9 +53,12 @@ def __init__(self, profile=None, global_endpoint=None, endpoint_map_file=None, - residency=None): + residency=None, + service_name=None): self._config_file = os.environ.get('ALERTLOGIC_CONFIG') self._endpoint_map = None + self._service_name = service_name + if self._config_file is None: self._config_file = almdrlib.constants.DEFAULT_CONFIG_FILE @@ -88,17 +93,34 @@ def __init__(self, os.environ.get('ALERTLOGIC_PROFILE') or \ almdrlib.constants.DEFAULT_PROFILE + self._service_name = \ + service_name or \ + os.environ.get('ALERTLOGIC_SERVICE_NAME') or \ + almdrlib.constants.DEFAULT_SERVICE_NAME + self._parser = configparser.ConfigParser() if self._read_config_file(): self._initialize_config() else: self._initialize_defaults() + self._init_al_env_credentials(service_name) + logger.debug("Finished configuraiton initialization. " + f"access_key_id={self._access_key_id}, " + f"account_id={self._account_id}, " + f"global_endpoint={self._global_endpoint}") + def _init_al_env_credentials(self, service_name): + try: + if self._access_key_id is None or self._secret_key is None: + service_name_key = f"{service_name}.aims_authc", + env = AlEnv(service_name_key) + self._access_key_id = env.get('access_key_id') + self._secret_key = env.get('secret_access_key') + except Exception as e: + logger.debug(f"Did not initialise aims credentials for {service_name} because {e}") + def _read_config_file(self): try: read_ok = self._parser.read(self._config_file) @@ -118,9 +140,7 @@ def _initialize_defaults(self): def _initialize_config(self): if self._access_key_id is None or self._secret_key is None: - self._access_key_id = self._get_config_option( - 'access_key_id', - None) + self._access_key_id = self._get_config_option('access_key_id', None) self._secret_key = self._get_config_option('secret_key', None) self._global_endpoint = \ diff --git a/almdrlib/constants.py b/almdrlib/constants.py index f5ade39..8ccfe5c 100644 --- a/almdrlib/constants.py +++ b/almdrlib/constants.py @@ -9,3 +9,4 @@ DEFAULT_GLOBAL_ENDPOINT = "production" DEFAULT_ENDPOINT_MAP_FILE = "endpoint_map.json" DEFAULT_RESIDENCY = "default" +DEFAULT_SERVICE_NAME = "almdrlib" diff --git a/almdrlib/environment.py b/almdrlib/environment.py new file mode 100644 index 0000000..c65708f --- /dev/null +++ b/almdrlib/environment.py @@ -0,0 +1,126 @@ +'''Cloud stored environment configuration +AlEnv class implements retrieval and formatting of configuration parameters from dynamodb in the standardized way. +AWS client used is configured as follows https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html +Therefore, AWS cli configuration would be automatically picked up. +The simpliest way is to use environment variables. +Alert Logic specific configuration is done via ALERTLOGIC_STACK_REGION and ALERTLOGIC_STACK_NAME env variables. + +Exceptions: +AlEnvException is a generic type for any AlEnv specific errors, derivatives are: +AlEnvConfigurationTableUnavailableException() - thrown when configuration table is not found +AlEnvAwsConfigurationException() - AWS credentials or configuration issue + +Network or io errors are not handled. + +Usage: +# export ALERTLOGIC_STACK_REGION=us-east-1 +# export ALERTLOGIC_STACK_NAME=integration +>> env = AlEnv("myapplication") +# Assuming parameter is stored in ddb as '"value"' +>> env.get("my_parameter") +'value' +>> env.get("my_parameter", format='raw') +'"value"' +# Assuming parameter is stored in ddb as '"1.0"' +>> env.get("my_parameter", type='float') +1.0 +>> env.get("my_parameter", type='integer') +1 +# Assuming parameter is stored in ddb as '"true"' +>> env.get("my_parameter", type='boolean') +True +''' + +import json +import boto3 +import os +import botocore + + +class AlEnvException(Exception): + pass + + +class AlEnvAwsConfigurationException(AlEnvException): + pass + + +class AlEnvConfigurationTableUnavailableException(AlEnvException): + pass + + +class AlEnv: + def __init__(self, application_name): + self.application_name = application_name + self.region = AlEnv._get_region() + self.stack_name = AlEnv._get_stack_name() + self.table_name = AlEnv._table_name(self.region, self.stack_name) + try: + self.dynamodb = boto3.resource('dynamodb') + self.table = self.dynamodb.Table(self.table_name) + self._table_date_time = self.table.creation_date_time + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == 'ResourceNotFoundException': + raise AlEnvConfigurationTableUnavailableException(self.table_name) + else: + raise AlEnvException(e) + except (botocore.exceptions.NoRegionError, botocore.exceptions.NoCredentialsError) as e: + raise AlEnvAwsConfigurationException(f'Please validate your AWS configuration: {e}') + + def get(self, key, default=None, format='decoded', type=None): + fetched_value = self.table.get_item(Key={"key": self._make_ddb_key(key)}).get('Item', {}).get('value') + converted = AlEnv._convert(fetched_value, format, type) + if converted is not None: + return converted + else: + return default + + def _make_ddb_key(self, option_key): + return f"{self.application_name}.{option_key}" + + @staticmethod + def _convert(value, format, type): + if format == 'raw' and value: + return AlEnv._format_value(value, type) + elif format == 'decoded' and value: + decoded = json.loads(value) + return AlEnv._format_value(decoded, type) + else: + return None + + @staticmethod + def _format_value(value, type): + if type == 'integer': + return int(value) + elif type == 'float': + return float(value) + elif type in ['boolean', 'bool']: + return AlEnv._to_bool(value) + else: + return value + + @staticmethod + def _to_bool(value): + if isinstance(value, bool): + return value + elif value == 'true': + return True + elif value == 'false': + return False + else: + raise ValueError('Provided value cannot be converted to boolean') + + @staticmethod + def _table_name(region, stack_name): + global_app = "global" + config_table = "settings" + environment_name = 'dev' + return f"{region}.{stack_name}.{environment_name}.{global_app}.{config_table}" + + @staticmethod + def _get_region(): + return os.environ.get('ALERTLOGIC_STACK_REGION', 'us-east-1') + + @staticmethod + def _get_stack_name(): + return os.environ.get('ALERTLOGIC_STACK_NAME', 'integration') diff --git a/requirements.txt b/requirements.txt index 2ff9ca0..39b74b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ configparser==4.0.2 pyyaml==5.1.2 jsonschema[format_nongpl]==3.2.0 git+https://github.com/crossnox/m2r@dev#egg=m2r +boto3>=1.16.57 \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt index b339016..fa2cfa1 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -15,3 +15,4 @@ sphinx_rtd_theme==0.4.3 sphinxcontrib_contentui==0.2.4 sphinx-paramlinks==0.4.1 git+https://github.com/crossnox/m2r@dev#egg=m2r +boto3>=1.16.57 \ No newline at end of file diff --git a/setup.py b/setup.py index 082b5b7..203a5b3 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,7 @@ 'pyyaml==5.1.2', 'jsonschema[format_nongpl]==3.2.0', 'm2r==0.2.1', + 'boto3>=1.16.57', definitions_dependency ] diff --git a/tests/test_environment.py b/tests/test_environment.py new file mode 100644 index 0000000..421b069 --- /dev/null +++ b/tests/test_environment.py @@ -0,0 +1,53 @@ +import unittest +from unittest.mock import MagicMock +from almdrlib.environment import AlEnv +import os +import boto3 + + +class MockDDBTable: + creation_date_time = '2005-08-09T18:31:42-03:30' + def __init__(self, tablename): + pass + + def get_item(self, **kwargs): + Key = kwargs.get('Key')['key'] + if Key == 'someapplication.strkey': + return {"Item": {"key": Key, "value": "\"strvalue\""}} + elif Key == 'someapplication.intkey': + return {"Item": {"key": Key, "value": "\"1\""}} + elif Key == 'someapplication.floatkey': + return {"Item": {"key": Key, "value": "\"1.0\""}} + elif Key == 'someapplication.boolkeytrue': + return {"Item": {"key": Key, "value": "\"true\""}} + elif Key == 'someapplication.boolkeyfalse': + return {"Item": {"key": Key, "value": "\"false\""}} + + +class MockBotoDDB: + Table = MockDDBTable + + def __init__(self, client_type): + pass + + +class TestAlEnv(unittest.TestCase): + def test_something(self): + boto3.resource = MagicMock(return_value=MockBotoDDB) + os.environ['ALERTLOGIC_STACK_REGION'] = 'us-west-1' + os.environ['ALERTLOGIC_STACK_NAME'] = 'production' + env = AlEnv("someapplication") + assert env.table_name == 'us-west-1.production.dev.global.settings' + assert env.get("strkey") == 'strvalue' + assert env.get("strkey", format='raw') == '"strvalue"' + assert env.get("intkey") == '1' + assert env.get("intkey", type='float') == 1.0 + assert env.get("boolkeyfalse") == 'false' + assert env.get("boolkeyfalse", type='boolean') is False + assert env.get("boolkeytrue", type='boolean') is True + assert env.get("floatkey") == '1.0' + assert env.get("floatkey", type='float') == 1.0 + + +if __name__ == '__main__': + unittest.main()