From bc197c12e7d8b225c4d931339a1b89917f2178dd Mon Sep 17 00:00:00 2001 From: Colin B Date: Thu, 7 Nov 2024 08:59:14 +0000 Subject: [PATCH 01/20] Update search query to handle ands within opensearch --- app/main/util/search_utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/main/util/search_utils.py b/app/main/util/search_utils.py index db679d16..7e0d0645 100644 --- a/app/main/util/search_utils.py +++ b/app/main/util/search_utils.py @@ -179,9 +179,8 @@ def build_dsl_search_query( "filter": filter_clauses, } }, - # set as {} until sorting ticket is in done - "sort": {}, - "_source": True, + "sort": sorting_orders, + "_source": {"exclude": ["*.keyword"]}, } From 11e90bcb765e8507d56c475957148de77b25f472 Mon Sep 17 00:00:00 2001 From: Colin B Date: Fri, 8 Nov 2024 16:28:19 +0000 Subject: [PATCH 02/20] split app loggeer, audit logger --- app/logger_config.py | 24 ++++++++++++++++++++++-- app/main/routes.py | 8 ++++---- app/main/util/render_utils.py | 6 +++--- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/app/logger_config.py b/app/logger_config.py index d8209399..e8939a49 100644 --- a/app/logger_config.py +++ b/app/logger_config.py @@ -21,6 +21,26 @@ def setup_logging(app): "[%(asctime)s] %(remote_addr)s requested %(url)s\n" "%(levelname)s in %(module)s: %(message)s" ) - default_handler.setFormatter(formatter) - app.logger.setLevel(logging.INFO) + # APP LOGGER + app_logger = logging.getLogger("app_logger") + app_handler = ( + logging.StreamHandler() + ) + app_handler.setFormatter(formatter) + app_logger.setLevel(logging.INFO) + app_logger.addHandler(app_handler) + + # AUDIT LOGGER + audit_logger = logging.getLogger("audit_logger") + audit_handler = ( + logging.StreamHandler() + ) + audit_handler.setFormatter(formatter) + audit_logger.setLevel(logging.INFO) + audit_logger.addHandler(audit_handler) + + app.logger.removeHandler(default_handler) + + app.audit_logger = audit_logger + app.app_logger = app_logger diff --git a/app/main/routes.py b/app/main/routes.py index 976729df..b33b33d4 100644 --- a/app/main/routes.py +++ b/app/main/routes.py @@ -681,7 +681,7 @@ def record(record_id: uuid.UUID): try: presigned_url = create_presigned_url(file) except Exception as e: - current_app.logger.info( + current_app.app_logger.info( f"Failed to create presigned url for document render non-javascript fallback {e}" ) @@ -726,7 +726,7 @@ def download_record(record_id: uuid.UUID): try: s3_file_object = s3.get_object(Bucket=bucket, Key=key) except Exception as e: - current_app.logger.error(f"Failed to get object from S3: {e}") + current_app.app_logger.error(f"Failed to get object from S3: {e}") abort(404) download_filename = file.FileName @@ -741,7 +741,7 @@ def download_record(record_id: uuid.UUID): file_content = s3_file_object["Body"].read() file_type = download_filename.split(".")[-1].lower() except Exception as e: - current_app.logger.error(f"Error reading S3 file content: {e}") + current_app.app_logger.error(f"Error reading S3 file content: {e}") abort(500) content_type = s3_file_object.get("ContentType", "application/octet-stream") @@ -760,7 +760,7 @@ def download_record(record_id: uuid.UUID): as_attachment=True, download_name=download_filename, ) - current_app.logger.info( + current_app.app_logger.info( json.dumps({"user_id": session["user_id"], "file": key}) ) return response diff --git a/app/main/util/render_utils.py b/app/main/util/render_utils.py index a221f50d..129e9b24 100644 --- a/app/main/util/render_utils.py +++ b/app/main/util/render_utils.py @@ -57,7 +57,7 @@ def get_download_filename(file): def create_presigned_url(file): file_extension = file.FileName.split(".")[-1].lower() if file_extension not in current_app.config["SUPPORTED_RENDER_EXTENSIONS"]: - current_app.logger.warning( + current_app.app_logger.warning( f"Rendering file format '{file_extension}' is not currently supported by AYR." ) return None @@ -88,7 +88,7 @@ def generate_pdf_manifest(record_id): try: presigned_url = create_presigned_url(file) except Exception as e: - current_app.logger.info( + current_app.app_logger.info( f"Failed to create presigned url for document render non-javascript fallback {e}" ) @@ -166,7 +166,7 @@ def generate_image_manifest(s3_file_object, record_id): try: presigned_url = create_presigned_url(file) except Exception as e: - current_app.logger.info( + current_app.app_logger.info( f"Failed to create presigned url for document render non-javascript fallback {e}" ) From 382c18a3120264e0a56485dc4f453cf3a05fbee9 Mon Sep 17 00:00:00 2001 From: Colin B Date: Fri, 8 Nov 2024 16:33:33 +0000 Subject: [PATCH 03/20] pre commit, readme --- README.md | 23 +++++++++++++++++------ app/logger_config.py | 8 ++------ 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 895147ea..8912f04e 100644 --- a/README.md +++ b/README.md @@ -599,20 +599,31 @@ This config includes: `setup_logging` is called during the initialization of the Flask app. ### Usage +We use two loggers in this application: -We can utilise the Flask logger by accessing Flask's `app.logger`. Since we define our routes with blueprints rather than the app directly, we can call access app through +- app_logger for application-related logs +- audit_logger for audit-specific logs + +Both are attached to the Flask app instance for easy access across the app. Since we define routes using blueprints instead of directly on the app, we access these loggers through Flask’s current_app. + +To use these loggers in your code, import current_app from Flask: ```python from flask import current_app ``` -for example: +Then, call the appropriate logger as follows: ```python -current_app.logger.info('Some info message') -current_app.logger.debug('Some debug message') -current_app.logger.warning('Some warning message') -current_app.logger.error('Some error message') +current_app.app_logger.info('Some info message') +current_app.app_logger.debug('Some debug message') +current_app.app_logger.warning('Some warning message') +current_app.app_logger.error('Some error message') + +current_app.audit_logger.info('Some info message') +current_app.audit_logger.debug('Some debug message') +current_app.audit_logger.warning('Some warning message') +current_app.audit_logger.error('Some error message') ``` ### Output diff --git a/app/logger_config.py b/app/logger_config.py index e8939a49..9d966873 100644 --- a/app/logger_config.py +++ b/app/logger_config.py @@ -24,18 +24,14 @@ def setup_logging(app): # APP LOGGER app_logger = logging.getLogger("app_logger") - app_handler = ( - logging.StreamHandler() - ) + app_handler = logging.StreamHandler() app_handler.setFormatter(formatter) app_logger.setLevel(logging.INFO) app_logger.addHandler(app_handler) # AUDIT LOGGER audit_logger = logging.getLogger("audit_logger") - audit_handler = ( - logging.StreamHandler() - ) + audit_handler = logging.StreamHandler() audit_handler.setFormatter(formatter) audit_logger.setLevel(logging.INFO) audit_logger.addHandler(audit_handler) From d2b0357d8bb4fdf949fdfe60fb0e7148229c1bea Mon Sep 17 00:00:00 2001 From: Colin B Date: Mon, 18 Nov 2024 11:49:48 +0000 Subject: [PATCH 04/20] updated sorting orders --- app/main/util/search_utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/main/util/search_utils.py b/app/main/util/search_utils.py index 7e0d0645..db679d16 100644 --- a/app/main/util/search_utils.py +++ b/app/main/util/search_utils.py @@ -179,8 +179,9 @@ def build_dsl_search_query( "filter": filter_clauses, } }, - "sort": sorting_orders, - "_source": {"exclude": ["*.keyword"]}, + # set as {} until sorting ticket is in done + "sort": {}, + "_source": True, } From 336390ab947c956012025b2d9aee4fc0f98d928c Mon Sep 17 00:00:00 2001 From: Colin B Date: Tue, 19 Nov 2024 19:31:06 +0000 Subject: [PATCH 05/20] update to split app & audit to diffrent log groups --- app/logger_config.py | 64 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 53 insertions(+), 11 deletions(-) diff --git a/app/logger_config.py b/app/logger_config.py index 9d966873..d57a6fc4 100644 --- a/app/logger_config.py +++ b/app/logger_config.py @@ -1,7 +1,8 @@ import logging +import boto3 +from botocore.exceptions import ClientError from flask import has_request_context, request -from flask.logging import default_handler class RequestFormatter(logging.Formatter): @@ -12,31 +13,72 @@ def format(self, record): else: record.url = None record.remote_addr = None - return super().format(record) -def setup_logging(app): +class CloudWatchHandler(logging.Handler): + def __init__(self, log_group, stream_name): + super().__init__() + self.client = boto3.client("logs") + self.log_group = log_group + self.stream_name = stream_name + + # Ensure log group and stream exist + try: + self.client.create_log_group(logGroupName=self.log_group) + except ClientError as e: + if e.response["Error"]["Code"] != "ResourceAlreadyExistsException": + raise e + + try: + self.client.create_log_stream( + logGroupName=self.log_group, logStreamName=self.stream_name + ) + except ClientError as e: + if e.response["Error"]["Code"] != "ResourceAlreadyExistsException": + raise e + + def emit(self, record): + try: + log_entry = self.format(record) + self.client.put_log_events( + logGroupName=self.log_group, + logStreamName=self.stream_name, + logEvents=[ + { + "timestamp": int(record.created * 1000), + "message": log_entry, + } + ], + ) + except ClientError as e: + print(f"Error sending log to CloudWatch: {e}") + + +# Configure Loggers with Formatter +def setup_loggers(app): formatter = RequestFormatter( "[%(asctime)s] %(remote_addr)s requested %(url)s\n" "%(levelname)s in %(module)s: %(message)s" ) - # APP LOGGER + # Application Logger -> CloudWatch Log Group: "app-logs" app_logger = logging.getLogger("app_logger") - app_handler = logging.StreamHandler() - app_handler.setFormatter(formatter) app_logger.setLevel(logging.INFO) + app_handler = CloudWatchHandler( + log_group="app-logs", stream_name="app-log-stream" + ) + app_handler.setFormatter(formatter) app_logger.addHandler(app_handler) - # AUDIT LOGGER + # Audit Logger -> CloudWatch Log Group: "audit-logs" audit_logger = logging.getLogger("audit_logger") - audit_handler = logging.StreamHandler() - audit_handler.setFormatter(formatter) audit_logger.setLevel(logging.INFO) + audit_handler = CloudWatchHandler( + log_group="audit-logs", stream_name="audit-log-stream" + ) + audit_handler.setFormatter(formatter) audit_logger.addHandler(audit_handler) - app.logger.removeHandler(default_handler) - app.audit_logger = audit_logger app.app_logger = app_logger From 535b6a6ad827ba5bdfcbd813d1fbe6449c8c2feb Mon Sep 17 00:00:00 2001 From: Colin B Date: Tue, 19 Nov 2024 19:34:04 +0000 Subject: [PATCH 06/20] fix logging setup --- app/logger_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/logger_config.py b/app/logger_config.py index d57a6fc4..c9394415 100644 --- a/app/logger_config.py +++ b/app/logger_config.py @@ -56,7 +56,7 @@ def emit(self, record): # Configure Loggers with Formatter -def setup_loggers(app): +def setup_logging(app): formatter = RequestFormatter( "[%(asctime)s] %(remote_addr)s requested %(url)s\n" "%(levelname)s in %(module)s: %(message)s" From 514403b1153984f88e2a542de0c32cd65da6913f Mon Sep 17 00:00:00 2001 From: Colin B Date: Tue, 19 Nov 2024 20:03:39 +0000 Subject: [PATCH 07/20] remove log creation, update for local --- app/logger_config.py | 62 ++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/app/logger_config.py b/app/logger_config.py index c9394415..fc5477e6 100644 --- a/app/logger_config.py +++ b/app/logger_config.py @@ -1,4 +1,5 @@ import logging +import os import boto3 from botocore.exceptions import ClientError @@ -23,21 +24,6 @@ def __init__(self, log_group, stream_name): self.log_group = log_group self.stream_name = stream_name - # Ensure log group and stream exist - try: - self.client.create_log_group(logGroupName=self.log_group) - except ClientError as e: - if e.response["Error"]["Code"] != "ResourceAlreadyExistsException": - raise e - - try: - self.client.create_log_stream( - logGroupName=self.log_group, logStreamName=self.stream_name - ) - except ClientError as e: - if e.response["Error"]["Code"] != "ResourceAlreadyExistsException": - raise e - def emit(self, record): try: log_entry = self.format(record) @@ -62,23 +48,37 @@ def setup_logging(app): "%(levelname)s in %(module)s: %(message)s" ) - # Application Logger -> CloudWatch Log Group: "app-logs" - app_logger = logging.getLogger("app_logger") - app_logger.setLevel(logging.INFO) - app_handler = CloudWatchHandler( - log_group="app-logs", stream_name="app-log-stream" - ) - app_handler.setFormatter(formatter) - app_logger.addHandler(app_handler) + if os.getenv("ENV") == "aws": + # Use CloudWatchHandler for non-local environments + app_logger = logging.getLogger("app_logger") + app_logger.setLevel(logging.INFO) + app_handler = CloudWatchHandler( + log_group="app-logs", stream_name="app-log-stream" + ) + app_handler.setFormatter(formatter) + app_logger.addHandler(app_handler) - # Audit Logger -> CloudWatch Log Group: "audit-logs" - audit_logger = logging.getLogger("audit_logger") - audit_logger.setLevel(logging.INFO) - audit_handler = CloudWatchHandler( - log_group="audit-logs", stream_name="audit-log-stream" - ) - audit_handler.setFormatter(formatter) - audit_logger.addHandler(audit_handler) + audit_logger = logging.getLogger("audit_logger") + audit_logger.setLevel(logging.INFO) + audit_handler = CloudWatchHandler( + log_group="audit-logs", stream_name="audit-log-stream" + ) + audit_handler.setFormatter(formatter) + audit_logger.addHandler(audit_handler) + + else: + # Use a simple console handler for local testing + app_logger = logging.getLogger("app_logger") + app_logger.setLevel(logging.INFO) + app_handler = logging.StreamHandler() + app_handler.setFormatter(formatter) + app_logger.addHandler(app_handler) + + audit_logger = logging.getLogger("audit_logger") + audit_logger.setLevel(logging.INFO) + audit_handler = logging.StreamHandler() + audit_handler.setFormatter(formatter) + audit_logger.addHandler(audit_handler) app.audit_logger = audit_logger app.app_logger = app_logger From 233c3a0e6cfe4a65d80f233b9699d2f34321a5d6 Mon Sep 17 00:00:00 2001 From: Colin B Date: Wed, 20 Nov 2024 13:08:24 +0000 Subject: [PATCH 08/20] remove comments --- app/logger_config.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/app/logger_config.py b/app/logger_config.py index fc5477e6..156e6fbc 100644 --- a/app/logger_config.py +++ b/app/logger_config.py @@ -41,7 +41,6 @@ def emit(self, record): print(f"Error sending log to CloudWatch: {e}") -# Configure Loggers with Formatter def setup_logging(app): formatter = RequestFormatter( "[%(asctime)s] %(remote_addr)s requested %(url)s\n" @@ -49,7 +48,6 @@ def setup_logging(app): ) if os.getenv("ENV") == "aws": - # Use CloudWatchHandler for non-local environments app_logger = logging.getLogger("app_logger") app_logger.setLevel(logging.INFO) app_handler = CloudWatchHandler( @@ -67,7 +65,6 @@ def setup_logging(app): audit_logger.addHandler(audit_handler) else: - # Use a simple console handler for local testing app_logger = logging.getLogger("app_logger") app_logger.setLevel(logging.INFO) app_handler = logging.StreamHandler() From 68dd13133d0bd22db2d203422771daa1c2eed85e Mon Sep 17 00:00:00 2001 From: Colin B Date: Thu, 21 Nov 2024 12:10:35 +0000 Subject: [PATCH 09/20] refactor logging setup --- app/logger_config.py | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/app/logger_config.py b/app/logger_config.py index 156e6fbc..ee4987ca 100644 --- a/app/logger_config.py +++ b/app/logger_config.py @@ -47,35 +47,27 @@ def setup_logging(app): "%(levelname)s in %(module)s: %(message)s" ) + app_logger = logging.getLogger("app_logger") + app_logger.setLevel(logging.INFO) + audit_logger = logging.getLogger("audit_logger") + audit_logger.setLevel(logging.INFO) + if os.getenv("ENV") == "aws": - app_logger = logging.getLogger("app_logger") - app_logger.setLevel(logging.INFO) app_handler = CloudWatchHandler( log_group="app-logs", stream_name="app-log-stream" ) - app_handler.setFormatter(formatter) - app_logger.addHandler(app_handler) - - audit_logger = logging.getLogger("audit_logger") - audit_logger.setLevel(logging.INFO) audit_handler = CloudWatchHandler( log_group="audit-logs", stream_name="audit-log-stream" ) - audit_handler.setFormatter(formatter) - audit_logger.addHandler(audit_handler) - else: - app_logger = logging.getLogger("app_logger") - app_logger.setLevel(logging.INFO) app_handler = logging.StreamHandler() - app_handler.setFormatter(formatter) - app_logger.addHandler(app_handler) - - audit_logger = logging.getLogger("audit_logger") - audit_logger.setLevel(logging.INFO) audit_handler = logging.StreamHandler() - audit_handler.setFormatter(formatter) - audit_logger.addHandler(audit_handler) + + app_handler.setFormatter(formatter) + audit_handler.setFormatter(formatter) + + app_logger.addHandler(app_handler) + audit_logger.addHandler(audit_handler) app.audit_logger = audit_logger app.app_logger = app_logger From 8167fbea24c62ab8c8fa3132c985279bbbeacd7f Mon Sep 17 00:00:00 2001 From: Colin B Date: Thu, 21 Nov 2024 12:26:52 +0000 Subject: [PATCH 10/20] update env check --- app/logger_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/logger_config.py b/app/logger_config.py index ee4987ca..d2d033e2 100644 --- a/app/logger_config.py +++ b/app/logger_config.py @@ -52,14 +52,14 @@ def setup_logging(app): audit_logger = logging.getLogger("audit_logger") audit_logger.setLevel(logging.INFO) - if os.getenv("ENV") == "aws": + if os.getenv("CONFIG_SOURCE") == "AWS_SECRETS_MANAGER": app_handler = CloudWatchHandler( log_group="app-logs", stream_name="app-log-stream" ) audit_handler = CloudWatchHandler( log_group="audit-logs", stream_name="audit-log-stream" ) - else: + elif os.getenv("CONFIG_SOURCE") == "ENVIRONMENT_VARIABLES": app_handler = logging.StreamHandler() audit_handler = logging.StreamHandler() From 651478ea0b6c53daaa2f00e0bcedfa5c0fafb95f Mon Sep 17 00:00:00 2001 From: Colin B Date: Thu, 21 Nov 2024 13:45:52 +0000 Subject: [PATCH 11/20] tests --- app/logger_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/logger_config.py b/app/logger_config.py index d2d033e2..5eade443 100644 --- a/app/logger_config.py +++ b/app/logger_config.py @@ -59,7 +59,7 @@ def setup_logging(app): audit_handler = CloudWatchHandler( log_group="audit-logs", stream_name="audit-log-stream" ) - elif os.getenv("CONFIG_SOURCE") == "ENVIRONMENT_VARIABLES": + else: app_handler = logging.StreamHandler() audit_handler = logging.StreamHandler() From 23eecb4e1b04767cfbcc3201ca60bca68d130554 Mon Sep 17 00:00:00 2001 From: Colin B Date: Mon, 25 Nov 2024 16:03:11 +0000 Subject: [PATCH 12/20] check test one --- app/logger_config.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/logger_config.py b/app/logger_config.py index 5eade443..4d044df1 100644 --- a/app/logger_config.py +++ b/app/logger_config.py @@ -53,15 +53,17 @@ def setup_logging(app): audit_logger.setLevel(logging.INFO) if os.getenv("CONFIG_SOURCE") == "AWS_SECRETS_MANAGER": + environment_name = os.getenv("ENVIRONMENT_NAME") + log_group_name = f"ayr-test-one-app-logs" app_handler = CloudWatchHandler( - log_group="app-logs", stream_name="app-log-stream" + log_group=log_group_name, stream_name="app-log-stream" ) audit_handler = CloudWatchHandler( - log_group="audit-logs", stream_name="audit-log-stream" + log_group=log_group_name, stream_name="audit-log-stream" ) else: app_handler = logging.StreamHandler() - audit_handler = logging.StreamHandler() + audit_handler = logging.StreamHandler() app_handler.setFormatter(formatter) audit_handler.setFormatter(formatter) From 2ad23ffba9edeabdadd60b97af5ec233d21d3d43 Mon Sep 17 00:00:00 2001 From: Colin B Date: Mon, 25 Nov 2024 16:03:44 +0000 Subject: [PATCH 13/20] ppre commit --- app/logger_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/logger_config.py b/app/logger_config.py index 4d044df1..a963594c 100644 --- a/app/logger_config.py +++ b/app/logger_config.py @@ -63,7 +63,7 @@ def setup_logging(app): ) else: app_handler = logging.StreamHandler() - audit_handler = logging.StreamHandler() + audit_handler = logging.StreamHandler() app_handler.setFormatter(formatter) audit_handler.setFormatter(formatter) From ac52fdb8fdc5027405aae715d48437dbd2e61147 Mon Sep 17 00:00:00 2001 From: Colin B Date: Mon, 25 Nov 2024 16:23:51 +0000 Subject: [PATCH 14/20] pre commit --- app/logger_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/logger_config.py b/app/logger_config.py index a963594c..83fb847f 100644 --- a/app/logger_config.py +++ b/app/logger_config.py @@ -53,8 +53,8 @@ def setup_logging(app): audit_logger.setLevel(logging.INFO) if os.getenv("CONFIG_SOURCE") == "AWS_SECRETS_MANAGER": - environment_name = os.getenv("ENVIRONMENT_NAME") - log_group_name = f"ayr-test-one-app-logs" + # environment_name = os.getenv("ENVIRONMENT_NAME") + log_group_name = "ayr-test-one-app-logs" app_handler = CloudWatchHandler( log_group=log_group_name, stream_name="app-log-stream" ) From c3aa99e82989edeafc9967ad987da07dbe41c68b Mon Sep 17 00:00:00 2001 From: Colin B Date: Tue, 26 Nov 2024 13:47:14 +0000 Subject: [PATCH 15/20] use env name var --- app/logger_config.py | 4 ++-- app/tests/test_config.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/app/logger_config.py b/app/logger_config.py index 83fb847f..25377c83 100644 --- a/app/logger_config.py +++ b/app/logger_config.py @@ -53,8 +53,8 @@ def setup_logging(app): audit_logger.setLevel(logging.INFO) if os.getenv("CONFIG_SOURCE") == "AWS_SECRETS_MANAGER": - # environment_name = os.getenv("ENVIRONMENT_NAME") - log_group_name = "ayr-test-one-app-logs" + environment_name = os.getenv("ENVIRONMENT_NAME") + log_group_name = f"ayr-{environment_name}-app-logs" app_handler = CloudWatchHandler( log_group=log_group_name, stream_name="app-log-stream" ) diff --git a/app/tests/test_config.py b/app/tests/test_config.py index 73406454..756c285a 100644 --- a/app/tests/test_config.py +++ b/app/tests/test_config.py @@ -272,6 +272,7 @@ def test_aws_secrets_manager_config_variable_not_set_error(monkeypatch): "DB_NAME": "test_db_name", "DB_SSL_ROOT_CERTIFICATE": "test_db_ssl_root_certificate", "DEFAULT_PAGE_SIZE": 10, + "ENVIRONMENT_NAME": "INT", } ) From 441b0caba387e3b654f4e7b0ee61217025f32c74 Mon Sep 17 00:00:00 2001 From: Colin B Date: Thu, 28 Nov 2024 15:46:42 +0000 Subject: [PATCH 16/20] Change to audit logger --- app/main/routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main/routes.py b/app/main/routes.py index b33b33d4..60f9bc55 100644 --- a/app/main/routes.py +++ b/app/main/routes.py @@ -760,7 +760,7 @@ def download_record(record_id: uuid.UUID): as_attachment=True, download_name=download_filename, ) - current_app.app_logger.info( + current_app.audit_logger.info( json.dumps({"user_id": session["user_id"], "file": key}) ) return response From 33cf388e540f708f63d1f86e76fddf567054cff5 Mon Sep 17 00:00:00 2001 From: Colin B Date: Thu, 28 Nov 2024 17:14:06 +0000 Subject: [PATCH 17/20] revise logger setup --- app/logger_config.py | 50 +++++++++++++++++++------------------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/app/logger_config.py b/app/logger_config.py index 25377c83..5ff168de 100644 --- a/app/logger_config.py +++ b/app/logger_config.py @@ -1,5 +1,4 @@ import logging -import os import boto3 from botocore.exceptions import ClientError @@ -41,35 +40,30 @@ def emit(self, record): print(f"Error sending log to CloudWatch: {e}") +def setup_logger(name, level, formatter): + """Helper function to set up a logger.""" + logger = logging.getLogger(name) + logger.setLevel(level) + handler = logging.StreamHandler() + handler.setFormatter(formatter) + logger.addHandler(handler) + return logger + + def setup_logging(app): - formatter = RequestFormatter( + """Set up loggers for the app.""" + audit_log_formatter = RequestFormatter( + "AUDIT_LOG\n" "[%(asctime)s] %(remote_addr)s requested %(url)s\n" "%(levelname)s in %(module)s: %(message)s" ) + app.audit_logger = setup_logger( + "audit_logger", logging.INFO, audit_log_formatter + ) - app_logger = logging.getLogger("app_logger") - app_logger.setLevel(logging.INFO) - audit_logger = logging.getLogger("audit_logger") - audit_logger.setLevel(logging.INFO) - - if os.getenv("CONFIG_SOURCE") == "AWS_SECRETS_MANAGER": - environment_name = os.getenv("ENVIRONMENT_NAME") - log_group_name = f"ayr-{environment_name}-app-logs" - app_handler = CloudWatchHandler( - log_group=log_group_name, stream_name="app-log-stream" - ) - audit_handler = CloudWatchHandler( - log_group=log_group_name, stream_name="audit-log-stream" - ) - else: - app_handler = logging.StreamHandler() - audit_handler = logging.StreamHandler() - - app_handler.setFormatter(formatter) - audit_handler.setFormatter(formatter) - - app_logger.addHandler(app_handler) - audit_logger.addHandler(audit_handler) - - app.audit_logger = audit_logger - app.app_logger = app_logger + app_log_formatter = RequestFormatter( + "APP_LOG\n" + "[%(asctime)s] %(remote_addr)s requested %(url)s\n" + "%(levelname)s in %(module)s: %(message)s" + ) + app.app_logger = setup_logger("app_logger", logging.INFO, app_log_formatter) From 667f6dee7351dcddd6eb8079dc5a55d03b12db30 Mon Sep 17 00:00:00 2001 From: Colin B Date: Thu, 28 Nov 2024 17:28:25 +0000 Subject: [PATCH 18/20] rem unused env var --- app/tests/test_config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/tests/test_config.py b/app/tests/test_config.py index 756c285a..73406454 100644 --- a/app/tests/test_config.py +++ b/app/tests/test_config.py @@ -272,7 +272,6 @@ def test_aws_secrets_manager_config_variable_not_set_error(monkeypatch): "DB_NAME": "test_db_name", "DB_SSL_ROOT_CERTIFICATE": "test_db_ssl_root_certificate", "DEFAULT_PAGE_SIZE": 10, - "ENVIRONMENT_NAME": "INT", } ) From e95284c9c9d4360ce47ad912a7eb4556b84923d4 Mon Sep 17 00:00:00 2001 From: Colin B Date: Thu, 28 Nov 2024 17:31:46 +0000 Subject: [PATCH 19/20] remove cloudwatch logger --- app/logger_config.py | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/app/logger_config.py b/app/logger_config.py index 5ff168de..53f886d4 100644 --- a/app/logger_config.py +++ b/app/logger_config.py @@ -16,30 +16,6 @@ def format(self, record): return super().format(record) -class CloudWatchHandler(logging.Handler): - def __init__(self, log_group, stream_name): - super().__init__() - self.client = boto3.client("logs") - self.log_group = log_group - self.stream_name = stream_name - - def emit(self, record): - try: - log_entry = self.format(record) - self.client.put_log_events( - logGroupName=self.log_group, - logStreamName=self.stream_name, - logEvents=[ - { - "timestamp": int(record.created * 1000), - "message": log_entry, - } - ], - ) - except ClientError as e: - print(f"Error sending log to CloudWatch: {e}") - - def setup_logger(name, level, formatter): """Helper function to set up a logger.""" logger = logging.getLogger(name) From f2e1c358836bb0a917f74f85576b7d683d595c6d Mon Sep 17 00:00:00 2001 From: Colin B Date: Thu, 28 Nov 2024 17:39:46 +0000 Subject: [PATCH 20/20] pre commit --- app/logger_config.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/logger_config.py b/app/logger_config.py index 53f886d4..99cc093b 100644 --- a/app/logger_config.py +++ b/app/logger_config.py @@ -1,7 +1,5 @@ import logging -import boto3 -from botocore.exceptions import ClientError from flask import has_request_context, request