From afc93ceca9359af7382143dadfb7ba0fe7e90808 Mon Sep 17 00:00:00 2001 From: Jake Rosenberg Date: Wed, 6 Nov 2024 16:28:16 -0600 Subject: [PATCH] task/WI-204: Enhanced logging for Audit Trails (#1482) * enhance logging for audit trails * include jobs post response in metrics log (#1483) --------- Co-authored-by: Sal Tijerina --- designsafe/apps/api/projects_v2/views.py | 223 +++++++++++++++++++ designsafe/apps/api/publications_v2/views.py | 62 ++++++ designsafe/apps/workspace/api/views.py | 47 ++-- designsafe/settings/common_settings.py | 2 +- 4 files changed, 306 insertions(+), 28 deletions(-) diff --git a/designsafe/apps/api/projects_v2/views.py b/designsafe/apps/api/projects_v2/views.py index 797cbc08f8..adac8a0e67 100644 --- a/designsafe/apps/api/projects_v2/views.py +++ b/designsafe/apps/api/projects_v2/views.py @@ -46,9 +46,11 @@ ) from designsafe.apps.api.projects_v2.schema_models.base import FileObj from designsafe.apps.api.decorators import tapis_jwt_login +from designsafe.apps.api.utils import get_client_ip logger = logging.getLogger(__name__) +metrics = logging.getLogger("metrics") def get_search_filter(query_string): @@ -74,6 +76,18 @@ def get(self, request: HttpRequest): # user = get_user_model().objects.get(username="ds_admin") user = request.user + metrics.info( + "Projects", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": "projects.listing", + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": {}, + }, + ) + if not request.user.is_authenticated: raise ApiException("Unauthenticated user", status=401) @@ -100,6 +114,18 @@ def post(self, request: HttpRequest): metadata_value = req_body.get("value", {}) # Projects are initialized as type None + metrics.info( + "Projects", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": "projects.create", + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": {"body": req_body}, + }, + ) + # increment project count prj_number = increment_workspace_count() # create metadata and graph @@ -123,6 +149,19 @@ class ProjectInstanceView(BaseApiView): def get(self, request: HttpRequest, project_id: str): """Return all project metadata for a project ID""" user = request.user + + metrics.info( + "Projects", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": "projects.detail", + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": {"project_id": project_id}, + }, + ) + if not request.user.is_authenticated: raise ApiException("Unauthenticated user", status=401) @@ -163,6 +202,19 @@ def put(self, request: HttpRequest, project_id: str): # Get the new value from the request data req_body = json.loads(request.body) + + metrics.info( + "Projects", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": "projects.change_project_type", + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": {"project_id": project, "body": req_body}, + }, + ) + new_value = req_body.get("value", {}) sensitive_data_option = req_body.get("sensitiveData", False) if sensitive_data_option: @@ -200,6 +252,19 @@ def patch(self, request: HttpRequest, project_id: str): ) from exc request_body = json.loads(request.body).get("patchMetadata", {}) + + metrics.info( + "Projects", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": "projects.patch_project", + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": {"project_id": project, "body": request_body}, + }, + ) + prev_metadata = BaseProject.model_validate(project.value) updated_project = patch_metadata(project.uuid, request_body) updated_metadata = BaseProject.model_validate(updated_project.value) @@ -231,6 +296,19 @@ def patch(self, request: HttpRequest, entity_uuid: str): ) request_body = json.loads(request.body).get("patchMetadata", {}) + + metrics.info( + "Projects", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": "projects.patch_metadata", + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": {"entity_uuid": entity_uuid, "body": request_body}, + }, + ) + logger.debug(request_body) patch_metadata(entity_uuid, request_body) return JsonResponse({"result": "OK"}) @@ -241,6 +319,18 @@ def delete(self, request: HttpRequest, entity_uuid: str): if not request.user.is_authenticated: raise ApiException("Unauthenticated user", status=401) + metrics.info( + "Projects", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": "projects.delete_entity", + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": {"entity_uuid": entity_uuid}, + }, + ) + entity_meta = ProjectMetadata.objects.get(uuid=entity_uuid) if user not in entity_meta.base_project.users.all(): raise ApiException( @@ -268,6 +358,19 @@ def post(self, request: HttpRequest, project_id: str): ) from exc req_body = json.loads(request.body) + + metrics.info( + "Projects", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": "projects.create_entity", + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": {"project_id": project_id, "body": req_body}, + }, + ) + value = req_body.get("value", {}) name = req_body.get("name", "") @@ -315,6 +418,18 @@ def put(self, request: HttpRequest, project_id: str): node_id = request_body.get("nodeId") order = request_body.get("order") + metrics.info( + "Projects", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": "projects.reorder_entities", + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": {"project_id": project_id, "body": request_body}, + }, + ) + if not node_id or order is None: raise ApiException("Node ID and new order must be specified", status=400) @@ -345,6 +460,22 @@ def post(self, request: HttpRequest, project_id, node_id): user = request.user entity_uuid = request_body.get("uuid") + metrics.info( + "Projects", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": "projects.delete_entity", + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": { + "project_id": project_id, + "node_id": node_id, + "body": request_body, + }, + }, + ) + if not entity_uuid: raise ApiException("Entity UUID must be provided", status=400) @@ -372,6 +503,19 @@ def post(self, request: HttpRequest, project_id, node_id): def delete(self, request: HttpRequest, project_id, node_id): """Remove a node from the project tree.""" user = request.user + + metrics.info( + "Projects", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": "projects.remove_node", + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": {"project_id": project_id, "node_id": node_id}, + }, + ) + if not request.user.is_authenticated: raise ApiException("Unauthenticated user", status=401) @@ -409,6 +553,22 @@ def patch(self, request: HttpRequest, project_id, entity_uuid): user = request.user + metrics.info( + "Projects", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": "projects.patch_files", + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": { + "project_id": project_id, + "entity_uuid": entity_uuid, + "body": file_obj_data, + }, + }, + ) + if not entity_uuid: raise ApiException("Entity UUID must be provided", status=400) @@ -451,6 +611,22 @@ def put(self, request: HttpRequest, project_id, entity_uuid): user = request.user + metrics.info( + "Projects", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": "projects.put_files", + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": { + "project_id": project_id, + "entity_uuid": entity_uuid, + "body": file_obj_data, + }, + }, + ) + if not entity_uuid: raise ApiException("Entity UUID must be provided", status=400) @@ -480,6 +656,22 @@ def delete(self, request: HttpRequest, project_id, entity_uuid, file_path): """Remove the association between a file and an entity.""" user = request.user + metrics.info( + "Projects", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": "projects.remove_file", + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": { + "project_id": project_id, + "entity_uuid": entity_uuid, + "file_path": file_path, + }, + }, + ) + if not entity_uuid: raise ApiException("Entity UUID must be provided", status=400) @@ -515,6 +707,22 @@ def put(self, request: HttpRequest, project_id, entity_uuid, file_path): user = request.user + metrics.info( + "Projects", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": "projects.set_file_tags", + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": { + "project_id": project_id, + "entity_uuid": entity_uuid, + "body": tag_names, + }, + }, + ) + if not entity_uuid: raise ApiException("Entity UUID must be provided", status=400) @@ -547,6 +755,21 @@ class ProjectEntityValidateView(BaseApiView): def post(self, request: HttpRequest, project_id): """validate a selection of entities to check publication-readiness.""" user = request.user + + metrics.info( + "Projects", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": "projects.validate_entities", + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": { + "project_id": project_id, + }, + }, + ) + if not request.user.is_authenticated: raise ApiException("Unauthenticated user", status=401) diff --git a/designsafe/apps/api/publications_v2/views.py b/designsafe/apps/api/publications_v2/views.py index 61f76eb1d9..7a9ffb5350 100644 --- a/designsafe/apps/api/publications_v2/views.py +++ b/designsafe/apps/api/publications_v2/views.py @@ -13,8 +13,10 @@ publish_project_async, amend_publication_async, ) +from designsafe.apps.api.utils import get_client_ip logger = logging.getLogger(__name__) +metrics = logging.getLogger("metrics") def handle_search(query_opts: dict, offset=0, limit=100): @@ -174,6 +176,18 @@ def get(self, request: HttpRequest): "data-type": request.GET.get("data-type", None), } + metrics.info( + "Publications", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": "publications.listing", + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": {"query": query_opts}, + }, + ) + has_query = any(query_opts.values()) if has_query: hits, total = handle_search(query_opts, offset, limit) @@ -219,6 +233,18 @@ def get(self, request: HttpRequest, project_id, version=None): except Publication.DoesNotExist as exc: raise ApiException(status=404, message="Publication not found.") from exc + metrics.info( + "Publications", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": "publications.detail", + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": {"project_id": project_id}, + }, + ) + pub_tree: nx.DiGraph = nx.node_link_graph(pub_meta.tree) file_tags = [] for file_tag_arr in [ @@ -244,6 +270,18 @@ def post(self, request: HttpRequest): request_body = json.loads(request.body) logger.debug(request_body) + metrics.info( + "Publications", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": "publications.publish", + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": {"body": request_body}, + }, + ) + project_id = request_body.get("projectId", None) entities_to_publish = request_body.get("entityUuids", None) @@ -274,6 +312,18 @@ def post(self, request: HttpRequest): request_body = json.loads(request.body) logger.debug(request_body) + metrics.info( + "Publications", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": "publications.version", + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": {"body": request_body}, + }, + ) + project_id = request_body.get("projectId", None) entities_to_publish = request_body.get("entityUuids", None) version_info = request_body.get("versionInfo", None) @@ -313,6 +363,18 @@ def post(self, request: HttpRequest): request_body = json.loads(request.body) logger.debug(request_body) + metrics.info( + "Publications", + extra={ + "user": request.user.username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": "publications.amend", + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": {"body": request_body}, + }, + ) + project_id = request_body.get("projectId", None) if not project_id: diff --git a/designsafe/apps/workspace/api/views.py b/designsafe/apps/workspace/api/views.py index 37ce7eb98c..cafbaa1735 100644 --- a/designsafe/apps/workspace/api/views.py +++ b/designsafe/apps/workspace/api/views.py @@ -175,7 +175,7 @@ def get(self, request, *args, **kwargs): extra={ "user": request.user.username, "sessionId": getattr(request.session, "session_key", ""), - "operation": "getAppsView", + "operation": "getApp", "agent": request.META.get("HTTP_USER_AGENT"), "ip": get_client_ip(request), "info": {"query": request.GET.dict()}, @@ -455,7 +455,7 @@ def get(self, request, *args, **kwargs): extra={ "user": request.user.username, "sessionId": getattr(request.session, "session_key", ""), - "operation": "getAppsTrayView", + "operation": "getApps", "agent": request.META.get("HTTP_USER_AGENT"), "ip": get_client_ip(request), "info": {"query": request.GET.dict()}, @@ -533,7 +533,6 @@ def get(self, request, operation=None): extra={ "user": request.user.username, "sessionId": getattr(request.session, "session_key", ""), - "view": "JobsView", "operation": operation, "agent": request.META.get("HTTP_USER_AGENT"), "ip": get_client_ip(request), @@ -638,7 +637,6 @@ def delete(self, request, *args, **kwargs): extra={ "user": request.user.username, "sessionId": getattr(request.session, "session_key", ""), - "view": "JobsView", "operation": "delete", "agent": request.META.get("HTTP_USER_AGENT"), "ip": get_client_ip(request), @@ -791,19 +789,6 @@ def post(self, request, *args, **kwargs): status=400, ) - METRICS.info( - "Jobs", - extra={ - "user": username, - "sessionId": getattr(request.session, "session_key", ""), - "view": "JobsView", - "operation": operation, - "agent": request.META.get("HTTP_USER_AGENT"), - "ip": get_client_ip(request), - "info": {"body": body}, - }, - ) - if operation != "submitJob": job_uuid = body.get("uuid") if job_uuid is None: @@ -812,18 +797,26 @@ def post(self, request, *args, **kwargs): status=400, ) tapis_operation = getattr(tapis.jobs, operation) - data = tapis_operation(jobUuid=job_uuid) + response = tapis_operation(jobUuid=job_uuid) - return JsonResponse( - { - "status": 200, - "response": data, - }, - encoder=BaseTapisResultSerializer, - ) + else: + # submit job + response = self._submit_job(request, body, tapis, username) - # submit job - response = self._submit_job(request, body, tapis, username) + METRICS.info( + "Jobs", + extra={ + "user": username, + "sessionId": getattr(request.session, "session_key", ""), + "operation": operation, + "agent": request.META.get("HTTP_USER_AGENT"), + "ip": get_client_ip(request), + "info": { + "body": body, + "response": response, + }, + }, + ) return JsonResponse( { diff --git a/designsafe/settings/common_settings.py b/designsafe/settings/common_settings.py index 237e365674..94c5eae3b4 100644 --- a/designsafe/settings/common_settings.py +++ b/designsafe/settings/common_settings.py @@ -427,7 +427,7 @@ 'metrics': { 'format': '[METRICS] %(levelname)s %(module)s %(name)s.%(funcName)s:%(lineno)s:' ' %(message)s user=%(user)s ip=%(ip)s agent=%(agent)s sessionId=%(sessionId)s op=%(operation)s' - ' info=%(info)s' + ' info=%(info)s timestamp=%(asctime)s portal=designsafe tenant=designsafe' }, }, 'handlers': {