Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

task/WI-224: Include tracking id in audit logs #1491

Merged
merged 4 commits into from
Dec 3, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions designsafe/apps/api/datafiles/handlers.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,19 +30,19 @@
}


def datafiles_get_handler(api, client, scheme, system, path, operation, username=None, **kwargs):
def datafiles_get_handler(api, client, scheme, system, path, operation, username=None, tapis_tracking_id=None, **kwargs):
if operation not in allowed_actions[scheme]:
raise PermissionDenied
op = getattr(operations_mapping[api], operation)

try:
return op(client, system, path, username=username, **kwargs)
return op(client, system, path, username=username, tapis_tracking_id=tapis_tracking_id, **kwargs)
except BaseTapyException as exc:
raise ApiException(message=exc.message, status=500) from exc


def datafiles_post_handler(api, username, client, scheme, system,
path, operation, body=None):
path, operation, body=None, tapis_tracking_id=None):

if operation not in allowed_actions[scheme]:
raise PermissionDenied
Expand All @@ -51,7 +51,7 @@ def datafiles_post_handler(api, username, client, scheme, system,
operation in notify_actions and notify(username, operation, '{} operation has started.'.format(operation.capitalize()), 'INFO', {})

try:
result = op(client, system, path, **body)
result = op(client, system, path, tapis_tracking_id=tapis_tracking_id, **body)
operation in notify_actions and notify(username, operation, '{} operation was successful.'.format(operation.capitalize()), 'SUCCESS', result)
return result
except Exception as exc:
Expand All @@ -60,15 +60,15 @@ def datafiles_post_handler(api, username, client, scheme, system,


def datafiles_put_handler(api, username, client, scheme, system,
path, operation, body=None):
path, operation, body=None, tapis_tracking_id=None):
if operation not in allowed_actions[scheme]:
raise PermissionDenied

op = getattr(operations_mapping[api], operation)
operation in notify_actions and notify(username, operation, '{} operation has started.'.format(operation.capitalize()), 'INFO', {})

try:
result = op(client, system, path, **body)
result = op(client, system, path, tapis_tracking_id=tapis_tracking_id, **body)
if operation == 'copy' and system != body.get('dest_system', None):
notify(username, operation, 'Your file transfer request has been received and will be processed shortly.'.format(operation.capitalize()), 'SUCCESS', result)
else:
Expand Down
38 changes: 23 additions & 15 deletions designsafe/apps/api/datafiles/operations/tapis_operations.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,8 @@ def listing(client, system, path, offset=0, limit=100, q=None, *args, **kwargs):
raw_listing = client.files.listFiles(systemId=system,
path=(path or '/'),
offset=int(offset),
limit=int(limit))
limit=int(limit),
headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id")})

try:
# Convert file objects to dicts for serialization.
Expand Down Expand Up @@ -83,7 +84,7 @@ def detail(client, system, path, *args, **kwargs):
"""
Retrieve the uuid for a file by parsing the query string in _links.metadata.href
"""
_listing = client.files.listFiles(systemId=system, path=urllib.parse.quote(path), offset=0, limit=1)
_listing = client.files.listFiles(systemId=system, path=urllib.parse.quote(path), offset=0, limit=1, headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id")})
f = _listing[0]
listing_res = {
'system': system,
Expand Down Expand Up @@ -229,7 +230,7 @@ def mkdir(client, system, path, dir_name):
return {"result": "OK"}


def move(client, src_system, src_path, dest_system, dest_path):
def move(client, src_system, src_path, dest_system, dest_path, *args, **kwargs):
"""
Move files and related file metadata

Expand Down Expand Up @@ -259,7 +260,8 @@ def move(client, src_system, src_path, dest_system, dest_path):
client.files.moveCopy(systemId=src_system,
path=src_path,
operation="MOVE",
newPath=dest_path_full)
newPath=dest_path_full,
headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id")})

move_file_meta_async.delay(src_system, src_path, dest_system, dest_path_full)

Expand All @@ -286,7 +288,7 @@ def move(client, src_system, src_path, dest_system, dest_path):
return {"result": "OK"}


def copy(client, src_system, src_path, dest_system, dest_path):
def copy(client, src_system, src_path, dest_system, dest_path, *args, **kwargs):
"""
Copy files and related file metadata

Expand Down Expand Up @@ -321,15 +323,16 @@ def copy(client, src_system, src_path, dest_system, dest_path):
copy_result = client.files.moveCopy(systemId=src_system,
path=src_path,
operation="COPY",
newPath=full_dest_path)
newPath=full_dest_path,
headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id")})
else:
src_url = f'tapis://{src_system}/{src_path}'
dest_url = f'tapis://{dest_system}/{full_dest_path}'

copy_response = client.files.createTransferTask(elements=[{
'sourceURI': src_url,
'destinationURI': dest_url
}])
}], headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id")})
copy_result = {
'uuid': copy_response.uuid,
'status': copy_response.status,
Expand Down Expand Up @@ -362,11 +365,12 @@ def copy(client, src_system, src_path, dest_system, dest_path):
return dict(copy_result)


def delete(client, system, path):
def delete(client, system, path, *args, **kwargs):
return client.files.delete(systemId=system,
filePath=urllib.parse.quote(path))
filePath=urllib.parse.quote(path),
headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id")})

def rename(client, system, path, new_name):
def rename(client, system, path, new_name, *args, **kwargs):
"""Renames a file. This is performed under the hood by moving the file to
the same parent folder but with a new name.

Expand Down Expand Up @@ -397,7 +401,8 @@ def rename(client, system, path, new_name):
client.files.moveCopy(systemId=system,
path=path,
operation="MOVE",
newPath=new_path)
newPath=new_path,
headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id")})

move_file_meta_async.delay(system, path, system, new_path)

Expand All @@ -415,7 +420,7 @@ def rename(client, system, path, new_name):



def trash(client, system, path, trash_path):
def trash(client, system, path, trash_path, *args, **kwargs):
"""Move a file to the .Trash folder.

Params
Expand All @@ -442,7 +447,7 @@ def trash(client, system, path, trash_path):
except tapipy.errors.NotFoundError:
mkdir(client, system, trash_root, trash_foldername)

resp = move(client, system, path, system, trash_path)
resp = move(client, system, path, system, trash_path, headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id")})

return resp

Expand Down Expand Up @@ -480,7 +485,10 @@ def upload(client, system, path, uploaded_file, webkit_relative_path=None, *args


dest_path = os.path.join(path.strip('/'), uploaded_file.name)
response_json = client.files.insert(systemId=system, path=dest_path, file=uploaded_file)
response_json = client.files.insert(systemId=system,
path=dest_path,
file=uploaded_file,
headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id")})
return {"result": "OK"}
agave_indexer.apply_async(kwargs={'systemId': system,
'filePath': path,
Expand Down Expand Up @@ -540,7 +548,7 @@ def preview(client, system, path, href="", max_uses=3, lifetime=600, *args, **kw
except FileMetaModel.DoesNotExist:
meta = {}

postit_result = client.files.createPostIt(systemId=system, path=path, allowedUses=max_uses, validSeconds=lifetime)
postit_result = client.files.createPostIt(systemId=system, path=path, allowedUses=max_uses, validSeconds=lifetime, headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id")})
url = postit_result.redeemUrl

if file_ext in settings.SUPPORTED_TEXT_PREVIEW_EXTS:
Expand Down
6 changes: 3 additions & 3 deletions designsafe/apps/api/datafiles/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ def get(self, request, api, operation=None, scheme='private', system=None, path=

try:
response = datafiles_get_handler(
api, client, scheme, system, path, operation, username=request.user.username, **request.GET.dict())
api, client, scheme, system, path, operation, tapis_tracking_id=f"portals.{request.session.session_key}", username=request.user.username, **request.GET.dict())
return JsonResponse(response)
except (BoxOAuthException, DropboxAuthError, GoogleAuthError):
raise resource_expired_handler(api)
Expand Down Expand Up @@ -104,7 +104,7 @@ def put(self, request, api, operation=None, scheme='private', system=None, path=
raise resource_unconnected_handler(api)

try:
response = datafiles_put_handler(api, request.user.username, client, scheme, system, path, operation, body=body)
response = datafiles_put_handler(api, request.user.username, client, scheme, system, path, operation, tapis_tracking_id=f"portals.{request.session.session_key}", body=body)
except HTTPError as e:
return JsonResponse({'message': str(e)}, status=e.response.status_code)

Expand Down Expand Up @@ -135,7 +135,7 @@ def post(self, request, api, operation=None, scheme='private', system=None, path
except AttributeError:
raise resource_unconnected_handler(api)

response = datafiles_post_handler(api, request.user.username, client, scheme, system, path, operation, body={**post_files, **post_body})
response = datafiles_post_handler(api, request.user.username, client, scheme, system, path, operation, tapis_tracking_id=f"portals.{request.session.session_key}", body={**post_files, **post_body})

return JsonResponse(response)

Expand Down
27 changes: 23 additions & 4 deletions designsafe/apps/workspace/api/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -561,7 +561,10 @@ def select(self, client, request):
"""Returns detailed information for a given job uuid"""

job_uuid = request.GET.get("uuid")
data = client.jobs.getJob(jobUuid=job_uuid)
data = client.jobs.getJob(
jobUuid=job_uuid,
headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"},
)

return data

Expand All @@ -584,6 +587,7 @@ def listing(self, client, request):
skip=skip,
orderBy="lastUpdated(desc),name(asc)",
select="allAttributes",
headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"},
**kwargs,
)
if isinstance(data, list):
Expand Down Expand Up @@ -626,6 +630,7 @@ def search(self, client, request):
orderBy="lastUpdated(desc),name(asc)",
request_body={"search": sql_queries},
select="allAttributes",
headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"},
)
return {"listing": data, "reachedEnd": len(data) < int(limit)}

Expand All @@ -645,7 +650,10 @@ def delete(self, request, *args, **kwargs):
)
tapis = request.user.tapis_oauth.client
job_uuid = request.GET.get("uuid")
data = tapis.jobs.hideJob(jobUuid=job_uuid)
data = tapis.jobs.hideJob(
jobUuid=job_uuid,
headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"},
)
return JsonResponse(
{
"status": 200,
Expand Down Expand Up @@ -755,6 +763,9 @@ def _submit_job(self, request, body, tapis, username):
tapis.files.mkdir(
systemId=exec_system_id,
path=f"{system['home_dir'].format(tasdir)}/.tap",
headers={
"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"
},
)

# Add webhook subscription for job status updates
Expand All @@ -771,7 +782,10 @@ def _submit_job(self, request, body, tapis, username):
]

logger.info(f"user: {username} is submitting job: {job_post}")
response = tapis.jobs.submitJob(**job_post)
response = tapis.jobs.submitJob(
**job_post,
headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"},
)
return response

def post(self, request, *args, **kwargs):
Expand All @@ -797,7 +811,12 @@ def post(self, request, *args, **kwargs):
status=400,
)
tapis_operation = getattr(tapis.jobs, operation)
response = tapis_operation(jobUuid=job_uuid)
response = tapis_operation(
jobUuid=job_uuid,
headers={
"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"
},
)

else:
# submit job
Expand Down
4 changes: 2 additions & 2 deletions designsafe/settings/common_settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -427,7 +427,7 @@
'metrics': {
'format': '[METRICS] %(levelname)s %(module)s %(name)s.%(funcName)s:%(lineno)s:'
' %(message)s user=%(user)s ip=%(ip)s agent=%(agent)s sessionId=%(sessionId)s op=%(operation)s'
' info=%(info)s timestamp=%(asctime)s portal=designsafe tenant=designsafe'
' info=%(info)s timestamp=%(asctime)s trackingId=portal.%(sessionId)s portal=designsafe tenant=designsafe'
},
},
'handlers': {
Expand Down Expand Up @@ -508,7 +508,7 @@
ALLOCATIONS_TO_EXCLUDE = (
os.environ.get("ALLOCATIONS_TO_EXCLUDE", "").split(",")
if os.environ.get("ALLOCATIONS_TO_EXCLUDE")
else ["DesignSafe-DCV"]
else ["DesignSafe-DCV,DesignSafe-Corral"]
)


Expand Down