From 07b5673a1f9be6ecd4c1ac9d62909db6138d7e36 Mon Sep 17 00:00:00 2001 From: Johannes Alneberg Date: Wed, 9 Mar 2022 11:24:55 +0100 Subject: [PATCH 001/293] Started testing --- .../666003748d14_change_active_nullable_and_default.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/migrations/versions/666003748d14_change_active_nullable_and_default.py b/migrations/versions/666003748d14_change_active_nullable_and_default.py index c183051b8..6596633fb 100644 --- a/migrations/versions/666003748d14_change_active_nullable_and_default.py +++ b/migrations/versions/666003748d14_change_active_nullable_and_default.py @@ -8,8 +8,6 @@ from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import mysql -from sqlalchemy.orm.session import Session -from dds_web.database import models # revision identifiers, used by Alembic. @@ -22,7 +20,8 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### session = Session(bind=op.get_bind()) - all_user_rows = session.query(models.User).all() + user_table = sa.sql.table("users", sa.sql.column("active")) + op.execute(user_table.update().where(user_table.c.active is None).values(active=False)) for user in all_user_rows: if user.active is None: user.active = False From 713575aaa28bca3ebe688053c7250ee93d43d855 Mon Sep 17 00:00:00 2001 From: Zishan Mirza Date: Wed, 9 Mar 2022 11:01:24 +0000 Subject: [PATCH 002/293] Rate limit: Error message This patch displays an error message when the user makes too many authentication requests. Signed-off-by: Zishan Mirza --- dds_web/templates/error.html | 4 ++++ dds_web/web/user.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/dds_web/templates/error.html b/dds_web/templates/error.html index 7030d8770..564e92d32 100644 --- a/dds_web/templates/error.html +++ b/dds_web/templates/error.html @@ -8,8 +8,12 @@ {% block body %}

+{% if error_code == 429 %} + API returned an error: Too many authentication requests in one hour +{% else %} An error occurred when processing your request. {{message}} +{% endif %}

Back to first page

diff --git a/dds_web/web/user.py b/dds_web/web/user.py index 888d7930c..15c2e526d 100644 --- a/dds_web/web/user.py +++ b/dds_web/web/user.py @@ -41,7 +41,7 @@ def bad_request(error): except AttributeError: message = "" flask.current_app.logger.error(f"{error.code}: {message}") - return flask.make_response(flask.render_template("error.html", message=message), error.code) + return flask.make_response(flask.render_template("error.html", message=message, error_code=error.code), error.code) #################################################################################################### From 7756f4f2970b539c48ab05f4bfca67eeccc87a79 Mon Sep 17 00:00:00 2001 From: Zishan Mirza Date: Wed, 9 Mar 2022 11:43:43 +0000 Subject: [PATCH 003/293] Rate limit: Error message Formatting. Signed-off-by: Zishan Mirza --- dds_web/web/user.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dds_web/web/user.py b/dds_web/web/user.py index 15c2e526d..cb9571d04 100644 --- a/dds_web/web/user.py +++ b/dds_web/web/user.py @@ -41,7 +41,9 @@ def bad_request(error): except AttributeError: message = "" flask.current_app.logger.error(f"{error.code}: {message}") - return flask.make_response(flask.render_template("error.html", message=message, error_code=error.code), error.code) + return flask.make_response( + flask.render_template("error.html", message=message, error_code=error.code), error.code + ) #################################################################################################### From 19c65e460931820a2e30104e2d6247e7dddb0db1 Mon Sep 17 00:00:00 2001 From: Zishan Mirza Date: Wed, 9 Mar 2022 13:07:22 +0000 Subject: [PATCH 004/293] Rate limit: Error message This patch updates the error message. Signed-off-by: Zishan Mirza --- dds_web/templates/error.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/templates/error.html b/dds_web/templates/error.html index 564e92d32..2a42a50f8 100644 --- a/dds_web/templates/error.html +++ b/dds_web/templates/error.html @@ -9,7 +9,7 @@

{% if error_code == 429 %} - API returned an error: Too many authentication requests in one hour + Too many authentication requests in one hour. Try again in a while. {% else %} An error occurred when processing your request. {{message}} From 85eb069ec200bf1dd5b672afa1d3d4d1138013f3 Mon Sep 17 00:00:00 2001 From: Johannes Alneberg Date: Wed, 9 Mar 2022 17:11:24 +0100 Subject: [PATCH 005/293] Migrations without session/model --- ...8d14_change_active_nullable_and_default.py | 7 +--- .../a5a40d843415_changed_sensitive.py | 42 ++++++++++++------- .../d117e6299dc9_remove_project_invites.py | 16 ++----- 3 files changed, 32 insertions(+), 33 deletions(-) diff --git a/migrations/versions/666003748d14_change_active_nullable_and_default.py b/migrations/versions/666003748d14_change_active_nullable_and_default.py index 6596633fb..486520499 100644 --- a/migrations/versions/666003748d14_change_active_nullable_and_default.py +++ b/migrations/versions/666003748d14_change_active_nullable_and_default.py @@ -19,13 +19,8 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - session = Session(bind=op.get_bind()) - user_table = sa.sql.table("users", sa.sql.column("active")) + user_table = sa.sql.table("users", sa.sql.column("active", mysql.TINYINT(display_width=1))) op.execute(user_table.update().where(user_table.c.active is None).values(active=False)) - for user in all_user_rows: - if user.active is None: - user.active = False - session.commit() op.alter_column("users", "active", existing_type=mysql.TINYINT(display_width=1), nullable=False) # ### end Alembic commands ### diff --git a/migrations/versions/a5a40d843415_changed_sensitive.py b/migrations/versions/a5a40d843415_changed_sensitive.py index 0911ba62d..11a167bf6 100644 --- a/migrations/versions/a5a40d843415_changed_sensitive.py +++ b/migrations/versions/a5a40d843415_changed_sensitive.py @@ -21,13 +21,23 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.add_column("projects", sa.Column("non_sensitive", sa.Boolean(), nullable=False)) - session = Session(bind=op.get_bind()) - all_project_rows = session.query(models.Project).all() - for proj in all_project_rows: - proj.non_sensitive = not proj.is_sensitive - session.commit() + # First create a nullable column (since a non-nullable would be impossible with data) + op.add_column( + "projects", sa.Column("non_sensitive", mysql.TINYINT(display_width=1), nullable=True) + ) + + project_table = sa.sql.table( + "projects", + sa.sql.column("non_sensitive", mysql.TINYINT(display_width=1)), + sa.sql.column("is_sensitive", mysql.TINYINT(display_width=1)), + ) + op.execute(project_table.update().values(non_sensitive=sa.not_(project_table.c.is_sensitive))) + # Now make the column non-nullable + op.alter_column( + "projects", "non_sensitive", existing_type=mysql.TINYINT(display_width=1), nullable=False + ) + op.drop_column("projects", "is_sensitive") # ### end Alembic commands ### @@ -35,16 +45,20 @@ def upgrade(): def downgrade(): # ### commands auto generated by Alembic - please adjust! ### op.add_column( + "projects", sa.Column("is_sensitive", mysql.TINYINT(display_width=1), nullable=True) + ) + + project_table = sa.sql.table( "projects", - sa.Column( - "is_sensitive", mysql.TINYINT(display_width=1), autoincrement=False, nullable=True - ), + sa.sql.column("non_sensitive", mysql.TINYINT(display_width=1)), + sa.sql.column("is_sensitive", mysql.TINYINT(display_width=1)), ) - session = Session(bind=op.get_bind()) - all_project_rows = session.query(models.Project).all() - for proj in all_project_rows: - proj.is_sensitive = not proj.non_sensitive - session.commit() + op.execute(project_table.update().values(is_sensitive=sa.not_(project_table.c.non_sensitive))) + # Now make the column non-nullable + op.alter_column( + "projects", "is_sensitive", existing_type=mysql.TINYINT(display_width=1), nullable=False + ) + op.drop_column("projects", "non_sensitive") # ### end Alembic commands ### diff --git a/migrations/versions/d117e6299dc9_remove_project_invites.py b/migrations/versions/d117e6299dc9_remove_project_invites.py index e727bd25e..1cb6c8cb4 100644 --- a/migrations/versions/d117e6299dc9_remove_project_invites.py +++ b/migrations/versions/d117e6299dc9_remove_project_invites.py @@ -19,25 +19,15 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### op.drop_table("projectinvites") - op.add_column("projectinvitekeys", sa.Column("owner", sa.Boolean(), nullable=False)) + op.add_column( + "projectinvitekeys", sa.Column("owner", mysql.TINYINT(display_width=1), nullable=False) + ) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### op.drop_column("projectinvitekeys", "owner") - op.create_table( - "apscheduler_jobs", - sa.Column("id", mysql.VARCHAR(length=191), nullable=False), - sa.Column("next_run_time", mysql.DOUBLE(asdecimal=True), nullable=True), - sa.Column("job_state", sa.BLOB(), nullable=False), - sa.PrimaryKeyConstraint("id"), - mysql_default_charset="utf8mb4", - mysql_engine="InnoDB", - ) - op.create_index( - "ix_apscheduler_jobs_next_run_time", "apscheduler_jobs", ["next_run_time"], unique=False - ) op.create_table( "projectinvites", sa.Column( From 87fb9302cede87c011d856b43d36e15229262a3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 10 Mar 2022 08:55:52 +0100 Subject: [PATCH 006/293] Fix missing/extra import, f-string --- dds_web/api/schemas/project_schemas.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dds_web/api/schemas/project_schemas.py b/dds_web/api/schemas/project_schemas.py index ecc38020d..872c0d926 100644 --- a/dds_web/api/schemas/project_schemas.py +++ b/dds_web/api/schemas/project_schemas.py @@ -8,13 +8,14 @@ import os # Installed +import botocore.client import flask import marshmallow import sqlalchemy # Own modules from dds_web import errors as ddserr -from dds_web import auth, db +from dds_web import auth from dds_web.database import models from dds_web.api import api_s3_connector from dds_web.api.schemas import sqlalchemyautoschemas @@ -145,7 +146,7 @@ def create_project(self, data, **kwargs): .one_or_none() ) if not unit_row: - raise ddserr.AccessDeniedError(message=f"Error: Your user is not associated to a unit.") + raise ddserr.AccessDeniedError(message="Error: Your user is not associated to a unit.") unit_row.counter = unit_row.counter + 1 if unit_row.counter else 1 data["public_id"] = "{}{:05d}".format(unit_row.internal_ref, unit_row.counter) From fe224b3b2ba1a6be423327bddcf9ca7be28d9a1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 10 Mar 2022 08:58:43 +0100 Subject: [PATCH 007/293] Remove extra import, raise exceptions from the old one --- dds_web/api/project.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index d0d3a41d1..f30032ed5 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -23,7 +23,6 @@ from dds_web.api.dds_decorators import ( logging_bind_request, dbsession, - args_required, json_required, handle_validation_errors, ) @@ -160,7 +159,7 @@ def post(self): project.project_statuses.append(add_status) if not project.is_active: # Deletes files (also commits session in the function - possibly refactor later) - removed = RemoveContents().delete_project_contents(project=project) + RemoveContents().delete_project_contents(project=project) delete_message = f"\nAll files in {project.public_id} deleted" if new_status in ["Deleted", "Archived"]: self.rm_project_user_keys(project=project) @@ -178,7 +177,7 @@ def post(self): ) as err: flask.current_app.logger.exception(err) db.session.rollback() - raise DatabaseError(message="Server Error: Status was not updated") + raise DatabaseError(message="Server Error: Status was not updated") from err # Mail users once project is made available if new_status == "Available" and send_email: @@ -404,7 +403,7 @@ def delete_project_contents(project): try: s3conn.remove_bucket() except botocore.client.ClientError as err: - raise DeletionError(message=str(err), project=project.public_id) + raise DeletionError(message=str(err), project=project.public_id) from err # If ok delete from database try: @@ -427,7 +426,7 @@ def delete_project_contents(project): "Project bucket contents were deleted, but they were not deleted from the " "database. Please contact SciLifeLab Data Centre." ), - ) + ) from sqlerr class CreateProject(flask_restful.Resource): @@ -455,7 +454,7 @@ def post(self): botocore.exceptions.ParamValidationError, ) as err: # For now just keeping the project row - raise S3ConnectionError(str(err)) + raise S3ConnectionError(str(err)) from err try: db.session.commit() From 3f6cb7defd7794825af9779056ec5116d8bd0528 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 10 Mar 2022 09:01:56 +0100 Subject: [PATCH 008/293] Remove a pile of unused imports --- dds_web/api/api_s3_connector.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/dds_web/api/api_s3_connector.py b/dds_web/api/api_s3_connector.py index 324aec81e..81c4ff855 100644 --- a/dds_web/api/api_s3_connector.py +++ b/dds_web/api/api_s3_connector.py @@ -7,28 +7,15 @@ # Standard library import logging import traceback -import pathlib -import json -import gc # Installed -import botocore -import flask -import sqlalchemy # Own modules from dds_web.api.dds_decorators import ( connect_cloud, bucket_must_exists, ) -from dds_web.errors import ( - BucketNotFoundError, - DatabaseError, - DeletionError, - S3ProjectNotFoundError, - S3InfoNotFoundError, - KeyNotFoundError, -) + from dds_web.database import models From c8134079a804563992dfd50c110da8ef65aa5f30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 10 Mar 2022 09:06:25 +0100 Subject: [PATCH 009/293] Unused imports, raise from --- dds_web/api/s3.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/dds_web/api/s3.py b/dds_web/api/s3.py index e06881074..5d7d3ce17 100644 --- a/dds_web/api/s3.py +++ b/dds_web/api/s3.py @@ -10,17 +10,14 @@ import flask_restful import flask import sqlalchemy -import marshmallow # Own modules from dds_web import auth from dds_web.api.api_s3_connector import ApiS3Connector -from dds_web.api.dds_decorators import logging_bind_request, args_required, handle_validation_errors +from dds_web.api.dds_decorators import logging_bind_request, handle_validation_errors from dds_web.errors import ( S3ProjectNotFoundError, DatabaseError, - DDSArgumentError, - MissingProjectIDError, ) from dds_web.api.schemas import project_schemas @@ -43,7 +40,7 @@ def get(self): try: sfsp_proj, keys, url, bucketname = ApiS3Connector(project=project).get_s3_info() except sqlalchemy.exc.SQLAlchemyError as sqlerr: - raise DatabaseError(message=str(sqlerr)) + raise DatabaseError(message=str(sqlerr)) from sqlerr if any(x is None for x in [url, keys, bucketname]): raise S3ProjectNotFoundError("No s3 info returned!") From 31598b9603e27329f1837fc2804fb02c1b0de9ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 10 Mar 2022 09:09:07 +0100 Subject: [PATCH 010/293] Remove useless try/except --- dds_web/api/schemas/file_schemas.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/dds_web/api/schemas/file_schemas.py b/dds_web/api/schemas/file_schemas.py index 8a6d5996d..a9309830d 100644 --- a/dds_web/api/schemas/file_schemas.py +++ b/dds_web/api/schemas/file_schemas.py @@ -106,19 +106,16 @@ def verify_file_not_exists(self, data, **kwargs): """Check that the file does not match anything already in the database.""" # Check that there is no such file in the database project = data.get("project_row") - try: - file = ( - models.File.query.filter( - sqlalchemy.and_( - models.File.name == sqlalchemy.func.binary(data.get("name")), - models.File.project_id == sqlalchemy.func.binary(project.id), - ) + file = ( + models.File.query.filter( + sqlalchemy.and_( + models.File.name == sqlalchemy.func.binary(data.get("name")), + models.File.project_id == sqlalchemy.func.binary(project.id), ) - .with_entities(models.File.id) - .one_or_none() ) - except sqlalchemy.exc.SQLAlchemyError: - raise + .with_entities(models.File.id) + .one_or_none() + ) if file: raise FileExistsError From 4be1363e89a85c2750f51337f215e5c55f1a4a75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 10 Mar 2022 09:23:35 +0100 Subject: [PATCH 011/293] Imports, direct raise, raise from --- dds_web/api/files.py | 90 ++++++++++++++++++++------------------------ 1 file changed, 41 insertions(+), 49 deletions(-) diff --git a/dds_web/api/files.py b/dds_web/api/files.py index 6b0d4d7c4..37511dab2 100644 --- a/dds_web/api/files.py +++ b/dds_web/api/files.py @@ -28,11 +28,10 @@ ) from dds_web.errors import ( AccessDeniedError, + BucketNotFoundError DatabaseError, DDSArgumentError, - EmptyProjectException, NoSuchFileError, - S3ConnectionError, ) from dds_web.api.schemas import file_schemas from dds_web.api.schemas import project_schemas @@ -97,7 +96,7 @@ def post(self): except sqlalchemy.exc.SQLAlchemyError as err: flask.current_app.logger.debug(err) db.session.rollback() - raise DatabaseError(f"Failed to add new file to database.") + raise DatabaseError("Failed to add new file to database.") from err return {"message": f"File '{new_file.name}' added to db."} @@ -179,7 +178,7 @@ def put(self): db.session.commit() except sqlalchemy.exc.SQLAlchemyError as err: db.session.rollback() - raise DatabaseError(f"Failed updating file information: {err}") + raise DatabaseError(f"Failed updating file information: {err}") from err return {"message": f"File '{file_info.get('name')}' updated in db."} @@ -207,7 +206,7 @@ def get(self): .all() ) except sqlalchemy.exc.SQLAlchemyError as err: - raise DatabaseError(f"Failed to get matching files in db: {err}") + raise DatabaseError(f"Failed to get matching files in db: {err}") from err # The files checked are not in the db if not matching_files or matching_files is None: @@ -249,12 +248,9 @@ def get(self): return {"num_items": 0, "message": f"The project {project.public_id} is empty."} # Get files and folders - try: - distinct_files, distinct_folders = self.items_in_subpath( - project=project, folder=subpath - ) - except DatabaseError: - raise + distinct_files, distinct_folders = self.items_in_subpath( + project=project, folder=subpath + ) # Collect file and folder info to return to CLI if distinct_files: @@ -298,7 +294,7 @@ def get_folder_size(self, project, folder_name): ) except sqlalchemy.exc.SQLAlchemyError as err: - raise DatabaseError(message=str(err)) + raise DatabaseError(message=str(err)) from err return file_info.sizeSum @@ -361,7 +357,7 @@ def items_in_subpath(project, folder="."): distinct_folders = list(split_paths) except sqlalchemy.exc.SQLAlchemyError as err: - raise DatabaseError(message=str(err)) + raise DatabaseError(message=str(err)) from err else: return distinct_files, distinct_folders @@ -481,41 +477,37 @@ def delete(self): # Remove folder(s) not_removed_dict, not_exist_list = ({}, []) - try: + with ApiS3Connector(project=project) as s3conn: + for x in flask.request.json: + # Get all files in the folder + try: + in_db, objects_to_delete = self.delete_folder(project=project, folder=x) + if not in_db: + not_exist_list.append(x) + raise FileNotFoundError( + "Could not find the specified folder in the database." + ) + except (sqlalchemy.exc.SQLAlchemyError, FileNotFoundError) as err: + db.session.rollback() + not_removed_dict[x] = str(err) + continue + + # Delete from s3 + try: + s3conn.remove_multiple(items=objects_to_delete) + except botocore.client.ClientError as err: + db.session.rollback() + not_removed_dict[x] = str(err) + continue + + # Commit to db if no error so far + try: + db.session.commit() + except sqlalchemy.exc.SQLAlchemyError as err: + db.session.rollback() + not_removed_dict[x] = str(err) + continue - with ApiS3Connector(project=project) as s3conn: - - for x in flask.request.json: - # Get all files in the folder - try: - in_db, objects_to_delete = self.delete_folder(project=project, folder=x) - if not in_db: - not_exist_list.append(x) - raise FileNotFoundError( - "Could not find the specified folder in the database." - ) - except (sqlalchemy.exc.SQLAlchemyError, FileNotFoundError) as err: - db.session.rollback() - not_removed_dict[x] = str(err) - continue - - # Delete from s3 - try: - s3conn.remove_multiple(items=objects_to_delete) - except botocore.client.ClientError as err: - db.session.rollback() - not_removed_dict[x] = str(err) - continue - - # Commit to db if no error so far - try: - db.session.commit() - except sqlalchemy.exc.SQLAlchemyError as err: - db.session.rollback() - not_removed_dict[x] = str(err) - continue - except (ValueError,): - raise return {"not_removed": not_removed_dict, "not_exists": not_exist_list} def delete_folder(self, project, folder): @@ -541,7 +533,7 @@ def delete_folder(self, project, folder): ) except sqlalchemy.exc.SQLAlchemyError as err: - raise DatabaseError(message=str(err)) + raise DatabaseError(message=str(err)) from err if files: exists = True @@ -657,7 +649,7 @@ def put(self): except sqlalchemy.exc.SQLAlchemyError as err: db.session.rollback() flask.current_app.logger.exception(str(err)) - raise DatabaseError("Update of file info failed.") + raise DatabaseError("Update of file info failed.") from err else: # flask.current_app.logger.debug("File %s updated", file_name) db.session.commit() From d11d5f0c4cad7d6c42b62eacecc3ebf1904a59f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 10 Mar 2022 09:28:28 +0100 Subject: [PATCH 012/293] Imports, non-fstring --- dds_web/api/user.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index e807ae7b3..64ed21a15 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -15,7 +15,6 @@ import flask_restful import flask_mail import itsdangerous -import marshmallow import structlog import sqlalchemy import http @@ -30,7 +29,6 @@ from dds_web.api.schemas import project_schemas, user_schemas, token_schemas from dds_web.api.dds_decorators import ( logging_bind_request, - args_required, json_required, handle_validation_errors, ) @@ -330,7 +328,7 @@ def add_to_project(whom, project, role, send_email=True): flask.current_app.logger.exception(err) db.session.rollback() raise ddserr.DatabaseError( - message=f"Server Error: User was not associated with the project" + message="Server Error: User was not associated with the project" ) # If project is already released and not expired, send mail to user From 94f7b01a478086662cd0e231c827f9696e48baf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 10 Mar 2022 09:35:02 +0100 Subject: [PATCH 013/293] Missing , --- dds_web/api/files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/api/files.py b/dds_web/api/files.py index 37511dab2..6e9c8be73 100644 --- a/dds_web/api/files.py +++ b/dds_web/api/files.py @@ -28,7 +28,7 @@ ) from dds_web.errors import ( AccessDeniedError, - BucketNotFoundError + BucketNotFoundError, DatabaseError, DDSArgumentError, NoSuchFileError, From 7e75119d74fa95d59751a86c6b88e5aa7609d146 Mon Sep 17 00:00:00 2001 From: Johannes Alneberg Date: Thu, 10 Mar 2022 09:39:20 +0100 Subject: [PATCH 014/293] Update the changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bedcb179c..0a1f4afb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - Add support for getting IPs from X-Forwarded-For ([#952](https://github.com/ScilifelabDataCentre/dds_web/pull/952)) - Relax requirements for usernames (wider length range, `.` and `-`) ([#943](https://github.com/ScilifelabDataCentre/dds_web/pull/943)) - Delay committing project to db until after the bucket has been created ([#967](https://github.com/ScilifelabDataCentre/dds_web/pull/967)) -- Fix logic for notification about sent email ([#963])(https://github.com/ScilifelabDataCentre/dds_web/pull/963)) +- Fix logic for notification about sent email ([#963])()) - Extended the `dds_web.api.dds_decorators.logging_bind_request` decorator to catch all not yet caught exceptions and make sure they will be logged ([#958](https://github.com/ScilifelabDataCentre/dds_web/pull/958)). - Increase the security of the session cookie using HTTPONLY and SECURE ([#972](https://github.com/ScilifelabDataCentre/dds_web/pull/972)) - Add role when listing project users ([#974](https://github.com/ScilifelabDataCentre/dds_web/pull/974)) @@ -45,3 +45,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe ## Sprint (2022-03-09 - 2022-03-23) - Introduce a separate error message if someone tried to add an unit user to projects individually. ([#1039](https://github.com/ScilifelabDataCentre/dds_web/pull/1039)) +- Future-proofing the migrations ([#1040](https://github.com/ScilifelabDataCentre/dds_web/pull/1040)) From fb2290b543a95681fe8b5362d5724333a4709c44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 10 Mar 2022 09:40:26 +0100 Subject: [PATCH 015/293] Remove unused import --- dds_web/run_app.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dds_web/run_app.py b/dds_web/run_app.py index 491c11ef3..58457e6bb 100644 --- a/dds_web/run_app.py +++ b/dds_web/run_app.py @@ -10,7 +10,6 @@ # Own modules from dds_web import create_app -from dds_web import auth #################################################################################################### From 7507e94597db11b94f94a94ce13cf36d289ccb6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 10 Mar 2022 09:56:35 +0100 Subject: [PATCH 016/293] Avoid redefining next, raise from, import grouping --- dds_web/web/user.py | 55 ++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/dds_web/web/user.py b/dds_web/web/user.py index 888d7930c..bff3a8239 100644 --- a/dds_web/web/user.py +++ b/dds_web/web/user.py @@ -9,21 +9,20 @@ # Installed import flask import werkzeug -from dds_web.api import db_tools import flask_login import itsdangerous import sqlalchemy # Own Modules -from dds_web import forms -from dds_web.database import models import dds_web.utils -from dds_web import db, limiter import dds_web.errors as ddserr +import dds_web.security +from dds_web import forms, db, limiter +from dds_web.api import db_tools from dds_web.api.dds_decorators import logging_bind_request from dds_web.api.schemas import user_schemas -import dds_web.security from dds_web.api.user import DeleteUser +from dds_web.database import models from dds_web.security.project_user_keys import update_user_keys_for_password_change auth_blueprint = flask.Blueprint("auth_blueprint", __name__) @@ -174,9 +173,9 @@ def confirm_2fa(): cancel_form = forms.Cancel2FAForm() - next = flask.request.args.get("next") + next_target = flask.request.args.get("next") # is_safe_url should check if the url is safe for redirects. - if next and not dds_web.utils.is_safe_url(next): + if next_target and not dds_web.utils.is_safe_url(next_target): return flask.abort(400) # Check user has initiated 2FA @@ -185,24 +184,24 @@ def confirm_2fa(): user = dds_web.security.auth.verify_token_no_data(token) except ddserr.AuthenticationError: flask.flash( - f"Error: Please initiate a log in before entering the one-time authentication code.", + "Error: Please initiate a log in before entering the one-time authentication code.", "warning", ) - return flask.redirect(flask.url_for("auth_blueprint.login", next=next)) + return flask.redirect(flask.url_for("auth_blueprint.login", next=next_target)) except Exception as e: flask.current_app.logger.exception(e) flask.flash( "Error: Second factor could not be validated due to an internal server error.", "danger", ) - return flask.redirect(flask.url_for("auth_blueprint.login", next=next)) + return flask.redirect(flask.url_for("auth_blueprint.login", next=next_target)) # Valid 2fa initiated token, but user does not exist (not never happen) or is inactive (could happen) # Currently same error for both, not vital, they get message to contact us if not user: flask.session.pop("2fa_initiated_token", None) flask.flash("Your account is not active. Contact Data Centre.", "warning") - return flask.redirect(flask.url_for("auth_blueprint.login", next=next)) + return flask.redirect(flask.url_for("auth_blueprint.login", next=next_target)) if form.validate_on_submit(): @@ -215,7 +214,7 @@ def confirm_2fa(): flask.flash("Invalid one-time code.", "warning") return flask.redirect( flask.url_for( - "auth_blueprint.confirm_2fa", form=form, cancel_form=cancel_form, next=next + "auth_blueprint.confirm_2fa", form=form, cancel_form=cancel_form, next=next_target ) ) @@ -225,11 +224,11 @@ def confirm_2fa(): # Remove token from session flask.session.pop("2fa_initiated_token", None) # Next is assured to be url_safe above - return flask.redirect(next or flask.url_for("pages.home")) + return flask.redirect(next_target or flask.url_for("pages.home")) else: return flask.render_template( - "user/confirm2fa.html", form=form, cancel_form=cancel_form, next=next + "user/confirm2fa.html", form=form, cancel_form=cancel_form, next=next_target ) @@ -242,17 +241,17 @@ def confirm_2fa(): def login(): """Initiate a login by validating username password and sending a authentication one-time code""" - next = flask.request.args.get("next") + next_target = flask.request.args.get("next") # is_safe_url should check if the url is safe for redirects. - if next and not dds_web.utils.is_safe_url(next): + if next_target and not dds_web.utils.is_safe_url(next_target): return flask.abort(400) # Redirect to next or index if user is already authenticated if flask_login.current_user.is_authenticated: - return flask.redirect(next or flask.url_for("pages.home")) + return flask.redirect(next_target or flask.url_for("pages.home")) # Display greeting message, if applicable - if next and re.search("confirm_deletion", next): + if next_target and re.search("confirm_deletion", next_target): flask.flash("Please log in to confirm your account deletion.", "warning") # Check if for is filled in and correctly (post) @@ -265,7 +264,7 @@ def login(): if not user or not user.verify_password(input_password=form.password.data): flask.flash("Invalid username or password.", "warning") return flask.redirect( - flask.url_for("auth_blueprint.login", next=next) + flask.url_for("auth_blueprint.login", next=next_target) ) # Try login again # Correct credentials still needs 2fa @@ -280,10 +279,10 @@ def login(): ) flask.session["2fa_initiated_token"] = token_2fa_initiated - return flask.redirect(flask.url_for("auth_blueprint.confirm_2fa", next=next)) + return flask.redirect(flask.url_for("auth_blueprint.confirm_2fa", next=next_target)) # Go to login form (get) - return flask.render_template("user/login.html", form=form, next=next) + return flask.render_template("user/login.html", form=form, next=next_target) @auth_blueprint.route("/logout", methods=["GET"]) @@ -463,7 +462,7 @@ def confirm_self_deletion(token): # Check that the email is registered on the current user: if email not in [email.email for email in flask_login.current_user.emails]: - msg = f"The email for user to be deleted is not registered on your account." + msg = "The email for user to be deleted is not registered on your account." flask.current_app.logger.warning( f"{msg} email: {email}: user: {flask_login.current_user}" ) @@ -474,18 +473,18 @@ def confirm_self_deletion(token): models.DeletionRequest.email == email ).first() - except itsdangerous.exc.SignatureExpired: + except itsdangerous.exc.SignatureExpired as exc: email = db_tools.remove_user_self_deletion_request(flask_login.current_user) raise ddserr.UserDeletionError( message=f"Deletion request for {email} has expired. Please login to the DDS and request deletion anew." - ) - except (itsdangerous.exc.BadSignature, itsdangerous.exc.BadTimeSignature): + ) from exc + except (itsdangerous.exc.BadSignature, itsdangerous.exc.BadTimeSignature) as exc: raise ddserr.UserDeletionError( - message=f"Confirmation link is invalid. No action has been performed." - ) + message="Confirmation link is invalid. No action has been performed." + ) from exc except sqlalchemy.exc.SQLAlchemyError as sqlerr: - raise ddserr.DatabaseError(message=sqlerr) + raise ddserr.DatabaseError(message=sqlerr) from sqlerr # Check if the user and the deletion request exists if deletion_request_row: From b632465efc71d0b40110d7ddbf8fa3fc4aed5929 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 10 Mar 2022 10:06:09 +0100 Subject: [PATCH 017/293] Imports, raise from --- dds_web/api/dds_decorators.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/dds_web/api/dds_decorators.py b/dds_web/api/dds_decorators.py index 5af30d792..705573759 100644 --- a/dds_web/api/dds_decorators.py +++ b/dds_web/api/dds_decorators.py @@ -16,19 +16,16 @@ import marshmallow # Own modules -from dds_web import db, auth +from dds_web import db from dds_web.errors import ( BucketNotFoundError, DatabaseError, DDSArgumentError, LoggedHTTPException, - NoSuchUserError, - AccessDeniedError, MissingJsonError, + S3ConnectionError, ) from dds_web.utils import get_username_or_request_ip -from dds_web.api.schemas import user_schemas, project_schemas -from dds_web.database import models # initiate logging action_logger = structlog.getLogger("actions") @@ -95,7 +92,7 @@ def make_commit(*args, **kwargs): try: db.session.commit() except sqlalchemy.exc.SQLAlchemyError as sqlerr: - raise DatabaseError(message=str(sqlerr), alt_message="Saving database changes failed.") + raise DatabaseError(message=str(sqlerr), alt_message="Saving database changes failed.") from sqlerr return result @@ -122,9 +119,9 @@ def init_resource(self, *args, **kwargs): aws_secret_access_key=self.keys["secret_key"], ) except sqlalchemy.exc.SQLAlchemyError as sqlerr: - raise DatabaseError(message=str(sqlerr)) + raise DatabaseError(message=str(sqlerr)) from sqlerr except botocore.client.ClientError as clierr: - raise S3ConnectionError(message=str(clierr)) + raise S3ConnectionError(message=str(clierr)) from sqlerr return func(self, *args, **kwargs) @@ -139,7 +136,7 @@ def check_bucket_exists(self, *args, **kwargs): try: self.resource.meta.client.head_bucket(Bucket=self.bucketname) except botocore.client.ClientError as err: - raise BucketNotFoundError(message=str(err)) + raise BucketNotFoundError(message=str(err)) from err return func(self, *args, **kwargs) From bef6a94257945d000f07f0d86e0a50ba15eacbdd Mon Sep 17 00:00:00 2001 From: Johannes Alneberg Date: Thu, 10 Mar 2022 10:31:16 +0100 Subject: [PATCH 018/293] Update CHANGELOG.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ina Odén Österbo <35953392+inaod568@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a1f4afb6..9ed249e45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - Add support for getting IPs from X-Forwarded-For ([#952](https://github.com/ScilifelabDataCentre/dds_web/pull/952)) - Relax requirements for usernames (wider length range, `.` and `-`) ([#943](https://github.com/ScilifelabDataCentre/dds_web/pull/943)) - Delay committing project to db until after the bucket has been created ([#967](https://github.com/ScilifelabDataCentre/dds_web/pull/967)) -- Fix logic for notification about sent email ([#963])()) +- Fix logic for notification about sent email ([#963](https://github.com/ScilifelabDataCentre/dds_web/pull/963)) - Extended the `dds_web.api.dds_decorators.logging_bind_request` decorator to catch all not yet caught exceptions and make sure they will be logged ([#958](https://github.com/ScilifelabDataCentre/dds_web/pull/958)). - Increase the security of the session cookie using HTTPONLY and SECURE ([#972](https://github.com/ScilifelabDataCentre/dds_web/pull/972)) - Add role when listing project users ([#974](https://github.com/ScilifelabDataCentre/dds_web/pull/974)) From 1ccfc19eb6407817d6758159896ca23d8d1b08c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 10 Mar 2022 10:58:57 +0100 Subject: [PATCH 019/293] Raise from --- dds_web/api/db_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/api/db_tools.py b/dds_web/api/db_tools.py index 872d03ffd..de5627767 100644 --- a/dds_web/api/db_tools.py +++ b/dds_web/api/db_tools.py @@ -36,6 +36,6 @@ def remove_user_self_deletion_request(user): db.session.commit() except sqlalchemy.exc.SQLAlchemyError as err: db.session.rollback() - raise DatabaseError(message=str(err)) + raise DatabaseError(message=str(err)) from err return email From cea765845b8b2be9c2153c14a83b06c07530dc7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 10 Mar 2022 11:20:33 +0100 Subject: [PATCH 020/293] Imports, avoid redefining built-ins, simplify if --- dds_web/utils.py | 59 ++++++++++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 32 deletions(-) diff --git a/dds_web/utils.py b/dds_web/utils.py index 9783aa453..70b829682 100644 --- a/dds_web/utils.py +++ b/dds_web/utils.py @@ -6,9 +6,7 @@ # Standard library import datetime -import json import os -import pathlib import re import urllib.parse @@ -17,21 +15,18 @@ import flask import flask_mail import flask_login -import sqlalchemy # # imports related to scheduling import atexit import werkzeug from apscheduler.schedulers import background import marshmallow -import flask_mail import wtforms # Own modules from dds_web.database import models from dds_web import auth, db, mail -import dds_web.errors as ddserr #################################################################################################### # VALIDATORS ########################################################################## VALIDATORS # @@ -40,62 +35,62 @@ # General ################################################################################ General # -def contains_uppercase(input): +def contains_uppercase(indata): """Verify that string contains at least one upper case letter.""" - if not re.search("[A-Z]", input): + if not re.search("[A-Z]", indata): raise marshmallow.ValidationError("Required: at least one upper case letter.") -def contains_lowercase(input): +def contains_lowercase(indata): """Verify that string contains at least one lower case letter.""" - if not re.search("[a-z]", input): + if not re.search("[a-z]", indata): raise marshmallow.ValidationError("Required: at least one lower case letter.") -def contains_digit_or_specialchar(input): +def contains_digit_or_specialchar(indata): """Verify that string contains at least one special character OR digit.""" - if not any(re.search(x, input) for x in ["[0-9]", "[#?!@$%^&*-]"]): + if not any(re.search(x, indata) for x in ["[0-9]", "[#?!@$%^&*-]"]): raise marshmallow.ValidationError( "Required: at least one digit OR a special character (#?!@$%^&*-)." ) -def contains_disallowed_characters(input): - """Inputs like <9f><98><80> cause issues in Project names etc.""" - disallowed = re.findall(r"[^\w\s]+", input) +def contains_disallowed_characters(indata): + """Indatas like <9f><98><80> cause issues in Project names etc.""" + disallowed = re.findall(r"[^\w\s]+", indata) if disallowed: disallowed = set(disallowed) # unique values chars = "characters" raise marshmallow.ValidationError( - f"The {chars if len(disallowed) > 1 else chars[:-1]} '{' '.join(disallowed)}' within '[italic]{input}[/italic]' {'are' if len(disallowed) > 1 else 'is'} not allowed." + f"The {chars if len(disallowed) > 1 else chars[:-1]} '{' '.join(disallowed)}' within '[italic]{indata}[/italic]' {'are' if len(disallowed) > 1 else 'is'} not allowed." ) -def email_not_taken(input): +def email_not_taken(indata): """Validator - verify that email is not taken. If used by marshmallow Schema, this validator should never raise an error since the email field should not be changable and if it is the form validator should catch it. """ - if email_in_db(email=input): + if email_in_db(email=indata): raise marshmallow.validate.ValidationError("The email is already taken by another user.") -def email_taken(input): +def email_taken(indata): """Validator - verify that email is taken.""" - if not email_in_db(email=input): + if not email_in_db(email=indata): raise marshmallow.validate.ValidationError( "There is no account with that email. To get an account, you need an invitation." ) -def username_not_taken(input): +def username_not_taken(indata): """Validate that username is not taken. If used by marshmallow Schema, this validator should never raise an error since the form validator should catch it. """ - if username_in_db(username=input): + if username_in_db(username=indata): raise marshmallow.validate.ValidationError( "That username is taken. Please choose a different one." ) @@ -118,7 +113,7 @@ def valid_user_role(specified_role): def username_contains_valid_characters(): def _username_contains_valid_characters(form, field): """Validate that the username contains valid characters.""" - if not valid_chars_in_username(input=field.data): + if not valid_chars_in_username(indata=field.data): raise wtforms.validators.ValidationError( "The username contains invalid characters. " "Usernames can only contain letters, digits and underscores (_)." @@ -138,7 +133,7 @@ def _password_contains_valid_characters(form, field): ] for val in validators: try: - val(input=field.data) + val(indata=field.data) except marshmallow.ValidationError as valerr: errors.append(str(valerr).strip(".")) @@ -152,7 +147,7 @@ def username_not_taken_wtforms(): def _username_not_taken(form, field): """Validate that the username is not taken already.""" try: - username_not_taken(input=field.data) + username_not_taken(indata=field.data) except marshmallow.validate.ValidationError as valerr: raise wtforms.validators.ValidationError(valerr) @@ -163,7 +158,7 @@ def email_not_taken_wtforms(): def _email_not_taken(form, field): """Validate that the email is not taken already.""" try: - email_not_taken(input=field.data) + email_not_taken(indata=field.data) except marshmallow.validate.ValidationError as valerr: raise wtforms.validators.ValidationError(valerr) @@ -174,7 +169,7 @@ def email_taken_wtforms(): def _email_taken(form, field): """Validate that the email exists.""" try: - email_taken(input=field.data) + email_taken(indata=field.data) except marshmallow.validate.ValidationError as valerr: raise wtforms.validators.ValidationError(valerr) @@ -186,9 +181,9 @@ def _email_taken(form, field): #################################################################################################### -def valid_chars_in_username(input): +def valid_chars_in_username(indata): """Check if the username contains only valid characters.""" - return False if re.search(r"^[a-zA-Z0-9_\.-]+$", input) == None else True + return bool(re.search(r"^[a-zA-Z0-9_\.-]+$", indata)) def email_in_db(email): @@ -302,7 +297,7 @@ def rate_limit_from_config(): @contextmanager -def working_directory(path, cleanup_after=False): +def working_directory(path): """Contexter for changing working directory""" current_path = os.getcwd() try: @@ -402,11 +397,11 @@ def scheduler_wrapper(): joblist = scheduler.get_jobs() jobid = [] for job in joblist: - id = getattr(job, "id") - jobid.append(id) + job_id = getattr(job, "id") + jobid.append(job_id) # Shut down the scheduler when exiting the app - atexit.register(lambda: scheduler.shutdown()) + atexit.register(scheduler.shutdown) # Print the currently scheduled jobs as verification: joblist = scheduler.get_jobs() From d88efdd5728aa3b6b92477be887108bc72d8fb35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 10 Mar 2022 11:22:29 +0100 Subject: [PATCH 021/293] Raise from --- dds_web/security/project_user_keys.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dds_web/security/project_user_keys.py b/dds_web/security/project_user_keys.py index e4cc3b214..66c1dfa2a 100644 --- a/dds_web/security/project_user_keys.py +++ b/dds_web/security/project_user_keys.py @@ -78,8 +78,8 @@ def __encrypt_project_private_key(owner, project_private_key): owner_public_key = serialization.load_der_public_key(owner.public_key) if isinstance(owner_public_key, asymmetric.rsa.RSAPublicKey): return __encrypt_with_rsa(project_private_key, owner_public_key) - except ValueError: - raise KeyOperationError(message="User public key could not be loaded!") + except ValueError as exc: + raise KeyOperationError(message="User public key could not be loaded!") from exc def __decrypt_project_private_key(user, token, encrypted_project_private_key): @@ -91,8 +91,8 @@ def __decrypt_project_private_key(user, token, encrypted_project_private_key): user_private_key = serialization.load_der_private_key(private_key_bytes, password=None) if isinstance(user_private_key, asymmetric.rsa.RSAPrivateKey): return __decrypt_with_rsa(encrypted_project_private_key, user_private_key) - except ValueError: - raise KeyOperationError(message="User private key could not be loaded!") + except ValueError as exc: + raise KeyOperationError(message="User private key could not be loaded!") from exc def obtain_project_private_key(user, project, token): From 179d1d13483d3666afcc0eb08544624fd0531fae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 10 Mar 2022 11:23:05 +0100 Subject: [PATCH 022/293] No-fstring --- dds_web/development/db_init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/development/db_init.py b/dds_web/development/db_init.py index e62ae125e..cfa70c732 100644 --- a/dds_web/development/db_init.py +++ b/dds_web/development/db_init.py @@ -85,7 +85,7 @@ def fill_db(): description="This is a test project. You will be able to upload to but NOT download " "from this project. Create a new project to test the entire system. ", pi="PI Name", - bucket=f"testbucket", + bucket="testbucket", ) project_1.project_statuses.append( From 2241f981f4ba2a88df313a2d45478dc7ae96b096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 10 Mar 2022 11:25:42 +0100 Subject: [PATCH 023/293] Fix import, f-string --- dds_web/development/factories.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dds_web/development/factories.py b/dds_web/development/factories.py index 3d960a5db..17e907a33 100644 --- a/dds_web/development/factories.py +++ b/dds_web/development/factories.py @@ -3,7 +3,7 @@ import random import flask -import dds_web.database.models as models +from dds_web.database import models from dds_web import db STATUSES_PER_PROJECT = 5 @@ -20,7 +20,7 @@ class Meta: sqlalchemy_session = db.session id = factory.Sequence(lambda n: n) - name = factory.Sequence(lambda n: "Unit {}".format(n)) + name = factory.Sequence(lambda n: f"Unit {n}") public_id = factory.Faker("uuid4") external_display_name = "Display Name" contact_email = "support@example.com" From 9b8c24af28c92c30c25ef73f159e6f065ba707ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 10 Mar 2022 11:28:18 +0100 Subject: [PATCH 024/293] Unused import, raise from --- dds_web/database/models.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dds_web/database/models.py b/dds_web/database/models.py index 10f34c28a..aacdb0849 100644 --- a/dds_web/database/models.py +++ b/dds_web/database/models.py @@ -14,7 +14,6 @@ import argon2 import flask_login import pathlib -from itsdangerous import TimedJSONWebSignatureSerializer as Serializer from cryptography.hazmat.primitives.twofactor import ( hotp as twofactor_hotp, InvalidToken as twofactor_InvalidToken, @@ -477,8 +476,8 @@ def verify_HOTP(self, token): try: hotp.verify(token, self.hotp_counter) - except twofactor_InvalidToken: - raise AuthenticationError("Invalid one-time authentication code.") + except twofactor_InvalidToken as exc: + raise AuthenticationError("Invalid one-time authentication code.") from exc # Token verified, increment counter to prohibit re-use self.hotp_counter += 1 From 219f80cf6027600d38604d9dc07b9ca660f33356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 10 Mar 2022 11:30:48 +0100 Subject: [PATCH 025/293] Raise from --- dds_web/security/auth.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/dds_web/security/auth.py b/dds_web/security/auth.py index e3d65f210..51701e252 100644 --- a/dds_web/security/auth.py +++ b/dds_web/security/auth.py @@ -166,15 +166,15 @@ def extract_token_invite_key(token): try: return invite, bytes.fromhex(claims.get("sen_con")) - except ValueError: - raise ValueError("Temporary key is expected be in hexadecimal digits for a byte string.") + except ValueError as exc: + raise ValueError("Temporary key is expected be in hexadecimal digits for a byte string.") from exc def obtain_current_encrypted_token(): try: return flask.request.headers["Authorization"].split()[1] - except KeyError: - raise TokenMissingError("Encrypted token is required but missing!") + except KeyError as exc: + raise TokenMissingError("Encrypted token is required but missing!") from exc def obtain_current_encrypted_token_claims(): @@ -220,7 +220,7 @@ def __verify_general_token(token): # jwcryopto.common.JWException is the base exception raised by jwcrypto, # and is raised when the token is malformed or invalid. flask.current_app.logger.exception(e) - raise AuthenticationError(message="Invalid token") + raise AuthenticationError(message="Invalid token") from e expiration_time = data.get("exp") # Use a hard check on top of the one from the dependency @@ -320,10 +320,10 @@ def verify_token_signature(token): try: jwttoken = jwt.JWT(key=key, jwt=token, algs=["HS256"]) return json.loads(jwttoken.claims) - except jwt.JWTExpired: + except jwt.JWTExpired as exc: # jwt dependency uses a 60 seconds leeway to check exp # it also prints out a stack trace for it, so we handle it here - raise AuthenticationError(message="Expired token") + raise AuthenticationError(message="Expired token") from exc except ValueError as exc: # "Token format unrecognized" raise AuthenticationError(message="Invalid token") from exc From c8303346c5bbc15b2838bc32b10b1af65c6fa8c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 10 Mar 2022 11:37:42 +0100 Subject: [PATCH 026/293] Fix formatting --- dds_web/api/dds_decorators.py | 4 +++- dds_web/api/files.py | 4 +--- dds_web/security/auth.py | 4 +++- dds_web/web/user.py | 5 ++++- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/dds_web/api/dds_decorators.py b/dds_web/api/dds_decorators.py index 705573759..8cfd0ebe5 100644 --- a/dds_web/api/dds_decorators.py +++ b/dds_web/api/dds_decorators.py @@ -92,7 +92,9 @@ def make_commit(*args, **kwargs): try: db.session.commit() except sqlalchemy.exc.SQLAlchemyError as sqlerr: - raise DatabaseError(message=str(sqlerr), alt_message="Saving database changes failed.") from sqlerr + raise DatabaseError( + message=str(sqlerr), alt_message="Saving database changes failed." + ) from sqlerr return result diff --git a/dds_web/api/files.py b/dds_web/api/files.py index 6e9c8be73..2ae2b9d34 100644 --- a/dds_web/api/files.py +++ b/dds_web/api/files.py @@ -248,9 +248,7 @@ def get(self): return {"num_items": 0, "message": f"The project {project.public_id} is empty."} # Get files and folders - distinct_files, distinct_folders = self.items_in_subpath( - project=project, folder=subpath - ) + distinct_files, distinct_folders = self.items_in_subpath(project=project, folder=subpath) # Collect file and folder info to return to CLI if distinct_files: diff --git a/dds_web/security/auth.py b/dds_web/security/auth.py index 51701e252..84c3cda93 100644 --- a/dds_web/security/auth.py +++ b/dds_web/security/auth.py @@ -167,7 +167,9 @@ def extract_token_invite_key(token): try: return invite, bytes.fromhex(claims.get("sen_con")) except ValueError as exc: - raise ValueError("Temporary key is expected be in hexadecimal digits for a byte string.") from exc + raise ValueError( + "Temporary key is expected be in hexadecimal digits for a byte string." + ) from exc def obtain_current_encrypted_token(): diff --git a/dds_web/web/user.py b/dds_web/web/user.py index bff3a8239..e5b67505f 100644 --- a/dds_web/web/user.py +++ b/dds_web/web/user.py @@ -214,7 +214,10 @@ def confirm_2fa(): flask.flash("Invalid one-time code.", "warning") return flask.redirect( flask.url_for( - "auth_blueprint.confirm_2fa", form=form, cancel_form=cancel_form, next=next_target + "auth_blueprint.confirm_2fa", + form=form, + cancel_form=cancel_form, + next=next_target, ) ) From f376351427ec9caae759e1b48d01d69fdd8fff49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Thu, 10 Mar 2022 15:59:20 +0100 Subject: [PATCH 027/293] catch key error --- dds_web/api/project.py | 34 ++++++++----- dds_web/api/user.py | 107 +++++++++++++++++++++++------------------ 2 files changed, 83 insertions(+), 58 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index d0d3a41d1..cf8eec0fd 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -601,9 +601,11 @@ def post(self): else: list_of_projects = [project] - self.give_project_access( + errors = self.give_project_access( project_list=list_of_projects, current_user=auth.current_user(), user=user ) + if errors: + return {"errors": errors} return {"message": f"Project access updated for user '{user.primary_email}'."} @@ -645,15 +647,23 @@ def verify_renew_access_permission(user, project): def give_project_access(project_list, current_user, user): """Give specific user project access.""" # Loop through and check that the project(s) is(are) active + fix_errors = {} for proj in project_list: - if proj.is_active: - project_keys_row = models.ProjectUserKeys.query.filter_by( - project_id=proj.id, user_id=user.username - ).one_or_none() - if not project_keys_row: - share_project_private_key( - from_user=current_user, - to_another=user, - project=proj, - from_user_token=dds_web.security.auth.obtain_current_encrypted_token(), - ) + try: + if proj.is_active: + project_keys_row = models.ProjectUserKeys.query.filter_by( + project_id=proj.id, user_id=user.username + ).one_or_none() + if not project_keys_row: + share_project_private_key( + from_user=current_user, + to_another=user, + project=proj, + from_user_token=dds_web.security.auth.obtain_current_encrypted_token(), + ) + except KeyNotFoundError as keyerr: + fix_errors[ + proj.public_id + ] = "You do not have access to this project. Please contact the responsible unit." + + return fix_errors diff --git a/dds_web/api/user.py b/dds_web/api/user.py index e807ae7b3..292ab2644 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -196,46 +196,52 @@ def invite_user(email, new_user_role, project=None, unit=None): # Compose and send email AddUser.compose_and_send_email_to_user(userobj=new_invite, mail_type="invite", link=link) - # Append invite to unit if applicable - if new_invite.role in ["Unit Admin", "Unit Personnel"]: - # TODO Change / move this later. This is just so that we can add an initial Unit Admin. - if auth.current_user().role == "Super Admin": - if unit: - unit_row = models.Unit.query.filter_by(public_id=unit).one_or_none() - if not unit_row: - raise ddserr.DDSArgumentError(message="Invalid unit publid id.") - - unit_row.invites.append(new_invite) - else: - raise ddserr.DDSArgumentError(message="Cannot invite this user.") - - if "Unit" in auth.current_user().role: - # Give new unit user access to all projects of the unit - auth.current_user().unit.invites.append(new_invite) - for unit_project in auth.current_user().unit.projects: - if unit_project.is_active: - share_project_private_key( - from_user=auth.current_user(), - to_another=new_invite, - from_user_token=dds_web.security.auth.obtain_current_encrypted_token(), - project=unit_project, - ) - - if not project: # specified project is disregarded for unituser invites - msg = f"{str(new_invite)} was successful." - else: - msg = f"{str(new_invite)} was successful, but specification for {str(project)} dropped. Unit Users have automatic access to projects of their unit." + try: + # Append invite to unit if applicable + if new_invite.role in ["Unit Admin", "Unit Personnel"]: + # TODO Change / move this later. This is just so that we can add an initial Unit Admin. + if auth.current_user().role == "Super Admin": + if unit: + unit_row = models.Unit.query.filter_by(public_id=unit).one_or_none() + if not unit_row: + raise ddserr.DDSArgumentError(message="Invalid unit publid id.") + + unit_row.invites.append(new_invite) + else: + raise ddserr.DDSArgumentError(message="Cannot invite this user.") + + if "Unit" in auth.current_user().role: + # Give new unit user access to all projects of the unit + auth.current_user().unit.invites.append(new_invite) + for unit_project in auth.current_user().unit.projects: + if unit_project.is_active: + share_project_private_key( + from_user=auth.current_user(), + to_another=new_invite, + from_user_token=dds_web.security.auth.obtain_current_encrypted_token(), + project=unit_project, + ) + + if not project: # specified project is disregarded for unituser invites + msg = f"{str(new_invite)} was successful." + else: + msg = f"{str(new_invite)} was successful, but specification for {str(project)} dropped. Unit Users have automatic access to projects of their unit." - else: - db.session.add(new_invite) - if project: - share_project_private_key( - from_user=auth.current_user(), - to_another=new_invite, - project=project, - from_user_token=dds_web.security.auth.obtain_current_encrypted_token(), - is_project_owner=new_user_role == "Project Owner", - ) + else: + db.session.add(new_invite) + if project: + share_project_private_key( + from_user=auth.current_user(), + to_another=new_invite, + project=project, + from_user_token=dds_web.security.auth.obtain_current_encrypted_token(), + is_project_owner=new_user_role == "Project Owner", + ) + except ddserr.KeyNotFoundError as keyerr: + return { + "message": "You do not have access to the specified project.", + "status": ddserr.AccessDeniedError.code.value, + } db.session.commit() msg = f"{str(new_invite)} was successful." @@ -316,13 +322,22 @@ def add_to_project(whom, project, role, send_email=True): ) ) - share_project_private_key( - from_user=auth.current_user(), - to_another=whom, - from_user_token=dds_web.security.auth.obtain_current_encrypted_token(), - project=project, - is_project_owner=is_owner, - ) + try: + share_project_private_key( + from_user=auth.current_user(), + to_another=whom, + from_user_token=dds_web.security.auth.obtain_current_encrypted_token(), + project=project, + is_project_owner=is_owner, + ) + except ddserr.KeyNotFoundError as keyerr: + return { + "message": ( + "You do not have access to the current project. To get access, " + "ask the a user within the responsible unit to grant you access." + ), + "status": ddserr.AccessDeniedError.code.value, + } try: db.session.commit() From 95cd8d2756737e512ddf1cfb5ed72e6b32f47ccc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Thu, 10 Mar 2022 16:21:37 +0100 Subject: [PATCH 028/293] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bedcb179c..aa7d46091 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,3 +45,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe ## Sprint (2022-03-09 - 2022-03-23) - Introduce a separate error message if someone tried to add an unit user to projects individually. ([#1039](https://github.com/ScilifelabDataCentre/dds_web/pull/1039)) +- Catch KeyNotFoundError when user tries to give access to a project they themselves do not have access to \ No newline at end of file From aa775c48a33b5b81925753d624f944586da5dd07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Thu, 10 Mar 2022 16:22:16 +0100 Subject: [PATCH 029/293] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa7d46091..4af912e42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,4 +45,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe ## Sprint (2022-03-09 - 2022-03-23) - Introduce a separate error message if someone tried to add an unit user to projects individually. ([#1039](https://github.com/ScilifelabDataCentre/dds_web/pull/1039)) -- Catch KeyNotFoundError when user tries to give access to a project they themselves do not have access to \ No newline at end of file +- Catch KeyNotFoundError when user tries to give access to a project they themselves do not have access to ([#1045](https://github.com/ScilifelabDataCentre/dds_web/pull/1045)) \ No newline at end of file From 24e97c72529dbe50470961ff9aa5767b71c484dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Thu, 10 Mar 2022 16:26:31 +0100 Subject: [PATCH 030/293] test adding space --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4af912e42..33f8bed6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,4 +45,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe ## Sprint (2022-03-09 - 2022-03-23) - Introduce a separate error message if someone tried to add an unit user to projects individually. ([#1039](https://github.com/ScilifelabDataCentre/dds_web/pull/1039)) -- Catch KeyNotFoundError when user tries to give access to a project they themselves do not have access to ([#1045](https://github.com/ScilifelabDataCentre/dds_web/pull/1045)) \ No newline at end of file +- Catch KeyNotFoundError when user tries to give access to a project they themselves do not have access to ([#1045](https://github.com/ScilifelabDataCentre/dds_web/pull/1045)) From 4fcc974b2865cc927e633de6a2c81564a944a996 Mon Sep 17 00:00:00 2001 From: Zishan Mirza Date: Thu, 10 Mar 2022 18:54:54 +0000 Subject: [PATCH 031/293] Rate limit: Error message This patch updates CHANGELOG.md. Signed-off-by: Zishan Mirza --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bedcb179c..d9548968c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,3 +45,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe ## Sprint (2022-03-09 - 2022-03-23) - Introduce a separate error message if someone tried to add an unit user to projects individually. ([#1039](https://github.com/ScilifelabDataCentre/dds_web/pull/1039)) +- Display an error message when the user makes too many authentication requests. ([#1034](https://github.com/ScilifelabDataCentre/dds_web/pull/1034)) From 428010918fe94644f5ef0142338e46f565d419dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Fri, 11 Mar 2022 06:20:24 +0100 Subject: [PATCH 032/293] Update dds_web/api/user.py --- dds_web/api/user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index 292ab2644..f1fca1f87 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -204,7 +204,7 @@ def invite_user(email, new_user_role, project=None, unit=None): if unit: unit_row = models.Unit.query.filter_by(public_id=unit).one_or_none() if not unit_row: - raise ddserr.DDSArgumentError(message="Invalid unit publid id.") + raise ddserr.DDSArgumentError(message="Invalid unit publid ID.") unit_row.invites.append(new_invite) else: From c2bc55de5d3af4083bd9102fbbd17f162d84a2d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Fri, 11 Mar 2022 06:37:52 +0100 Subject: [PATCH 033/293] moved try except and return a message dict --- dds_web/api/user.py | 68 ++++++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index 292ab2644..a0d557684 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -196,40 +196,46 @@ def invite_user(email, new_user_role, project=None, unit=None): # Compose and send email AddUser.compose_and_send_email_to_user(userobj=new_invite, mail_type="invite", link=link) - try: - # Append invite to unit if applicable - if new_invite.role in ["Unit Admin", "Unit Personnel"]: - # TODO Change / move this later. This is just so that we can add an initial Unit Admin. - if auth.current_user().role == "Super Admin": - if unit: - unit_row = models.Unit.query.filter_by(public_id=unit).one_or_none() - if not unit_row: - raise ddserr.DDSArgumentError(message="Invalid unit publid id.") - - unit_row.invites.append(new_invite) - else: - raise ddserr.DDSArgumentError(message="Cannot invite this user.") - - if "Unit" in auth.current_user().role: - # Give new unit user access to all projects of the unit - auth.current_user().unit.invites.append(new_invite) - for unit_project in auth.current_user().unit.projects: - if unit_project.is_active: + projects_not_shared = {} + # Append invite to unit if applicable + if new_invite.role in ["Unit Admin", "Unit Personnel"]: + # TODO Change / move this later. This is just so that we can add an initial Unit Admin. + if auth.current_user().role == "Super Admin": + if unit: + unit_row = models.Unit.query.filter_by(public_id=unit).one_or_none() + if not unit_row: + raise ddserr.DDSArgumentError(message="Invalid unit publid id.") + + unit_row.invites.append(new_invite) + else: + raise ddserr.DDSArgumentError(message="Cannot invite this user.") + + if "Unit" in auth.current_user().role: + # Give new unit user access to all projects of the unit + auth.current_user().unit.invites.append(new_invite) + for unit_project in auth.current_user().unit.projects: + if unit_project.is_active: + try: share_project_private_key( from_user=auth.current_user(), to_another=new_invite, from_user_token=dds_web.security.auth.obtain_current_encrypted_token(), project=unit_project, ) + except ddserr.KeyNotFoundError as keyerr: + projects_not_shared[ + unit_project.public_id + ] = "You do not have access to the specified project." - if not project: # specified project is disregarded for unituser invites - msg = f"{str(new_invite)} was successful." - else: - msg = f"{str(new_invite)} was successful, but specification for {str(project)} dropped. Unit Users have automatic access to projects of their unit." + if not project: # specified project is disregarded for unituser invites + msg = f"{str(new_invite)} was successful." + else: + msg = f"{str(new_invite)} was successful, but specification for {str(project)} dropped. Unit Users have automatic access to projects of their unit." - else: - db.session.add(new_invite) - if project: + else: + db.session.add(new_invite) + if project: + try: share_project_private_key( from_user=auth.current_user(), to_another=new_invite, @@ -237,11 +243,10 @@ def invite_user(email, new_user_role, project=None, unit=None): from_user_token=dds_web.security.auth.obtain_current_encrypted_token(), is_project_owner=new_user_role == "Project Owner", ) - except ddserr.KeyNotFoundError as keyerr: - return { - "message": "You do not have access to the specified project.", - "status": ddserr.AccessDeniedError.code.value, - } + except ddserr.KeyNotFoundError as keyerr: + projects_not_shared[ + unit_project.public_id + ] = "You do not have access to the specified project." db.session.commit() msg = f"{str(new_invite)} was successful." @@ -250,6 +255,7 @@ def invite_user(email, new_user_role, project=None, unit=None): "email": new_invite.email, "message": msg, "status": http.HTTPStatus.OK, + "errors": projects_not_shared, } @staticmethod From 2eee60f057336d5902d3cdf73f6d36b725c623ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Fri, 11 Mar 2022 06:50:31 +0100 Subject: [PATCH 034/293] only send email any worked --- dds_web/api/user.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index a0d557684..3f6c2331c 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -193,10 +193,8 @@ def invite_user(email, new_user_role, project=None, unit=None): "status": http.HTTPStatus.INTERNAL_SERVER_ERROR, } - # Compose and send email - AddUser.compose_and_send_email_to_user(userobj=new_invite, mail_type="invite", link=link) - projects_not_shared = {} + goahead = True # Append invite to unit if applicable if new_invite.role in ["Unit Admin", "Unit Personnel"]: # TODO Change / move this later. This is just so that we can add an initial Unit Admin. @@ -227,6 +225,9 @@ def invite_user(email, new_user_role, project=None, unit=None): unit_project.public_id ] = "You do not have access to the specified project." + # Check if any succeeded + goahead = len(projects_not_shared) == len(auth.current_user().unit.projects) + if not project: # specified project is disregarded for unituser invites msg = f"{str(new_invite)} was successful." else: @@ -247,10 +248,18 @@ def invite_user(email, new_user_role, project=None, unit=None): projects_not_shared[ unit_project.public_id ] = "You do not have access to the specified project." + goahead = False db.session.commit() - msg = f"{str(new_invite)} was successful." + # Compose and send email + if goahead: + AddUser.compose_and_send_email_to_user( + userobj=new_invite, mail_type="invite", link=link + ) + msg = f"{str(new_invite)} was successful." + else: + msg = f"The user could not be added to at least one project." return { "email": new_invite.email, "message": msg, From 9100c213e44b6476dba081f380ddf1e2458aaf89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Fri, 11 Mar 2022 06:51:41 +0100 Subject: [PATCH 035/293] try except for commit --- dds_web/api/user.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index 3f6c2331c..56f4d4508 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -250,8 +250,10 @@ def invite_user(email, new_user_role, project=None, unit=None): ] = "You do not have access to the specified project." goahead = False - db.session.commit() - + try: + db.session.commit() + except sqlalchemy.exc.SQLAlchemyError as sqlerr: + raise ddserr.DatabaseError(message=str(sqlerr)) # Compose and send email if goahead: AddUser.compose_and_send_email_to_user( @@ -260,6 +262,7 @@ def invite_user(email, new_user_role, project=None, unit=None): msg = f"{str(new_invite)} was successful." else: msg = f"The user could not be added to at least one project." + return { "email": new_invite.email, "message": msg, From 10fd0b3eb7dface3ea172a040001fde43a527bd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Fri, 11 Mar 2022 06:55:05 +0100 Subject: [PATCH 036/293] changed goahead --- dds_web/api/user.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index 56f4d4508..181f2ccd2 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -194,7 +194,7 @@ def invite_user(email, new_user_role, project=None, unit=None): } projects_not_shared = {} - goahead = True + goahead = False # Append invite to unit if applicable if new_invite.role in ["Unit Admin", "Unit Personnel"]: # TODO Change / move this later. This is just so that we can add an initial Unit Admin. @@ -205,6 +205,7 @@ def invite_user(email, new_user_role, project=None, unit=None): raise ddserr.DDSArgumentError(message="Invalid unit publid id.") unit_row.invites.append(new_invite) + goahead = True else: raise ddserr.DDSArgumentError(message="Cannot invite this user.") @@ -224,9 +225,8 @@ def invite_user(email, new_user_role, project=None, unit=None): projects_not_shared[ unit_project.public_id ] = "You do not have access to the specified project." - - # Check if any succeeded - goahead = len(projects_not_shared) == len(auth.current_user().unit.projects) + else: + goahead = True if not project: # specified project is disregarded for unituser invites msg = f"{str(new_invite)} was successful." @@ -248,7 +248,8 @@ def invite_user(email, new_user_role, project=None, unit=None): projects_not_shared[ unit_project.public_id ] = "You do not have access to the specified project." - goahead = False + else: + goahead = True try: db.session.commit() From 72cbaab9241837453fa25345354331f69418e815 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Fri, 11 Mar 2022 06:56:01 +0100 Subject: [PATCH 037/293] added space --- dds_web/api/user.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index 181f2ccd2..fcefd091a 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -255,6 +255,7 @@ def invite_user(email, new_user_role, project=None, unit=None): db.session.commit() except sqlalchemy.exc.SQLAlchemyError as sqlerr: raise ddserr.DatabaseError(message=str(sqlerr)) + # Compose and send email if goahead: AddUser.compose_and_send_email_to_user( From 1891da1fba7977b350bb235d70aec784b32a390a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Fri, 11 Mar 2022 07:04:33 +0100 Subject: [PATCH 038/293] moved token in tests --- tests/test_user_add.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_user_add.py b/tests/test_user_add.py index 2bbecabdf..367ba1353 100644 --- a/tests/test_user_add.py +++ b/tests/test_user_add.py @@ -109,8 +109,8 @@ def test_add_user_with_unitadmin_and_invalid_email(client): def test_add_user_with_unitadmin(client): + token = tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client) with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: - token = tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client) response = client.post( tests.DDSEndpoint.USER_ADD, headers=token, @@ -199,8 +199,8 @@ def test_add_unit_user_with_unitadmin(client): def test_add_user_with_superadmin(client): + token = tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client) with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: - token = tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client) response = client.post( tests.DDSEndpoint.USER_ADD, headers=token, From c8f9a6a6b1867f1d6fed73f6c01b8e212c3eec6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Fri, 11 Mar 2022 07:57:18 +0100 Subject: [PATCH 039/293] moved token again --- tests/test_user_add.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_user_add.py b/tests/test_user_add.py index 367ba1353..ea3b32797 100644 --- a/tests/test_user_add.py +++ b/tests/test_user_add.py @@ -109,8 +109,8 @@ def test_add_user_with_unitadmin_and_invalid_email(client): def test_add_user_with_unitadmin(client): - token = tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client) with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: + token = tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client) response = client.post( tests.DDSEndpoint.USER_ADD, headers=token, @@ -198,9 +198,8 @@ def test_add_unit_user_with_unitadmin(client): def test_add_user_with_superadmin(client): - - token = tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client) with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: + token = tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client) response = client.post( tests.DDSEndpoint.USER_ADD, headers=token, From c6f4b1df016dabe190dbb0797246e034955e8822 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Fri, 11 Mar 2022 12:57:49 +0100 Subject: [PATCH 040/293] change messages --- dds_web/api/user.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index fcefd091a..0c0a944e9 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -246,29 +246,37 @@ def invite_user(email, new_user_role, project=None, unit=None): ) except ddserr.KeyNotFoundError as keyerr: projects_not_shared[ - unit_project.public_id + project.public_id ] = "You do not have access to the specified project." else: goahead = True - - try: - db.session.commit() - except sqlalchemy.exc.SQLAlchemyError as sqlerr: - raise ddserr.DatabaseError(message=str(sqlerr)) + else: + goahead = True # Compose and send email + status_code = http.HTTPStatus.OK if goahead: + try: + db.session.commit() + except sqlalchemy.exc.SQLAlchemyError as sqlerr: + raise ddserr.DatabaseError(message=str(sqlerr)) + AddUser.compose_and_send_email_to_user( userobj=new_invite, mail_type="invite", link=link ) msg = f"{str(new_invite)} was successful." else: - msg = f"The user could not be added to at least one project." + msg = ( + f"The user could not be added to the project(s)." + if projects_not_shared + else "Unknown error!" + ) + " The invite did not succeed." + status_code = ddserr.InviteError.code.value return { "email": new_invite.email, "message": msg, - "status": http.HTTPStatus.OK, + "status": status_code, "errors": projects_not_shared, } From 359334b39b45a3b692fd26d9244ca2d2fec8cfb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Fri, 11 Mar 2022 13:34:09 +0100 Subject: [PATCH 041/293] reformat messages --- dds_web/api/user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index 0c0a944e9..3d37b16ba 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -224,7 +224,7 @@ def invite_user(email, new_user_role, project=None, unit=None): except ddserr.KeyNotFoundError as keyerr: projects_not_shared[ unit_project.public_id - ] = "You do not have access to the specified project." + ] = "You do not have access to the project(s)" else: goahead = True From ada86d5220078fee9a093d647fdf6616ad49c805 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Fri, 11 Mar 2022 15:41:02 +0100 Subject: [PATCH 042/293] adding more tests --- dds_web/api/user.py | 6 +- tests/test_user_add.py | 1297 ++++++++++++++++++++-------------------- 2 files changed, 669 insertions(+), 634 deletions(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index 3d37b16ba..10ad15042 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -6,9 +6,11 @@ # Standard library import os +from re import T import smtplib import time import datetime +from tkinter.tix import Tree # Installed import flask @@ -207,7 +209,9 @@ def invite_user(email, new_user_role, project=None, unit=None): unit_row.invites.append(new_invite) goahead = True else: - raise ddserr.DDSArgumentError(message="Cannot invite this user.") + raise ddserr.DDSArgumentError( + message="You need to specify a unit to invite a Unit Personnel or Unit Admin." + ) if "Unit" in auth.current_user().role: # Give new unit user access to all projects of the unit diff --git a/tests/test_user_add.py b/tests/test_user_add.py index ea3b32797..f7deb4714 100644 --- a/tests/test_user_add.py +++ b/tests/test_user_add.py @@ -1,3 +1,4 @@ +from urllib import response import dds_web import flask_mail import http @@ -43,663 +44,693 @@ } # Inviting Users ################################################################# Inviting Users # -def test_add_user_with_researcher(client): - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["researchuser"]).token(client), - json=first_new_user, - ) - assert response.status_code == http.HTTPStatus.FORBIDDEN - invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() - assert invited_user is None - - -def test_add_user_with_unituser_no_role(client): - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), - json=first_new_email, - ) - assert response.status_code == http.HTTPStatus.BAD_REQUEST - invited_user = models.Invite.query.filter_by(email=first_new_email["email"]).one_or_none() - assert invited_user is None - - -def test_add_user_with_unitadmin_with_extraargs(client): - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), - json=first_new_user_extra_args, - ) - assert response.status_code == http.HTTPStatus.OK - invited_user = models.Invite.query.filter_by( - email=first_new_user_extra_args["email"] - ).one_or_none() - assert invited_user - - -def test_add_user_with_unitadmin_and_invalid_role(client): - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), - json=first_new_user_invalid_role, - ) - assert response.status_code == http.HTTPStatus.BAD_REQUEST - invited_user = models.Invite.query.filter_by( - email=first_new_user_invalid_role["email"] - ).one_or_none() - assert invited_user is None - - -def test_add_user_with_unitadmin_and_invalid_email(client): - with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), - json=first_new_user_invalid_email, - ) - assert response.status_code == http.HTTPStatus.BAD_REQUEST - # An email is always sent when receiving the partial token - mock_mail_send.assert_called_once() +# def test_add_user_with_researcher(client): +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["researchuser"]).token(client), +# json=first_new_user, +# ) +# assert response.status_code == http.HTTPStatus.FORBIDDEN +# invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() +# assert invited_user is None + + +# def test_add_user_with_unituser_no_role(client): +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), +# json=first_new_email, +# ) +# assert response.status_code == http.HTTPStatus.BAD_REQUEST +# invited_user = models.Invite.query.filter_by(email=first_new_email["email"]).one_or_none() +# assert invited_user is None + + +# def test_add_user_with_unitadmin_with_extraargs(client): +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), +# json=first_new_user_extra_args, +# ) +# assert response.status_code == http.HTTPStatus.OK +# invited_user = models.Invite.query.filter_by( +# email=first_new_user_extra_args["email"] +# ).one_or_none() +# assert invited_user + + +# def test_add_user_with_unitadmin_and_invalid_role(client): +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), +# json=first_new_user_invalid_role, +# ) +# assert response.status_code == http.HTTPStatus.BAD_REQUEST +# invited_user = models.Invite.query.filter_by( +# email=first_new_user_invalid_role["email"] +# ).one_or_none() +# assert invited_user is None + + +# def test_add_user_with_unitadmin_and_invalid_email(client): +# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), +# json=first_new_user_invalid_email, +# ) +# assert response.status_code == http.HTTPStatus.BAD_REQUEST +# # An email is always sent when receiving the partial token +# mock_mail_send.assert_called_once() + +# invited_user = models.Invite.query.filter_by( +# email=first_new_user_invalid_email["email"] +# ).one_or_none() +# assert invited_user is None + + +# def test_add_user_with_unitadmin(client): +# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: +# token = tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client) +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=token, +# json=first_new_user, +# ) +# # One mail sent for partial token and one for the invite +# assert mock_mail_send.call_count == 2 + +# assert response.status_code == http.HTTPStatus.OK + +# invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() +# assert invited_user +# assert invited_user.email == first_new_user["email"] +# assert invited_user.role == first_new_user["role"] + +# assert invited_user.nonce is not None +# assert invited_user.public_key is not None +# assert invited_user.private_key is not None +# assert invited_user.project_invite_keys == [] + +# # Repeating the invite should not send a new invite: +# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=token, +# json=first_new_user, +# ) +# # No new mail should be sent for the token and neither for an invite +# assert mock_mail_send.call_count == 0 +# assert response.status_code == http.HTTPStatus.BAD_REQUEST +# message = response.json.get("message") +# assert "user was already added to the system" in message + + +# def test_add_unit_user_with_unitadmin(client): + +# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: +# token = tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client) +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=token, +# json=new_unit_user, +# ) +# # One mail sent for partial token and one for the invite +# assert mock_mail_send.call_count == 2 + +# assert response.status_code == http.HTTPStatus.OK + +# invited_user = models.Invite.query.filter_by(email=new_unit_user["email"]).one_or_none() +# assert invited_user +# assert invited_user.email == new_unit_user["email"] +# assert invited_user.role == new_unit_user["role"] + +# assert invited_user.nonce is not None +# assert invited_user.public_key is not None +# assert invited_user.private_key is not None + +# project_invite_keys = invited_user.project_invite_keys +# number_of_asserted_projects = 0 +# for project_invite_key in project_invite_keys: +# if ( +# project_invite_key.project.public_id == "public_project_id" +# or project_invite_key.project.public_id == "unused_project_id" +# or project_invite_key.project.public_id == "restricted_project_id" +# or project_invite_key.project.public_id == "second_public_project_id" +# or project_invite_key.project.public_id == "file_testing_project" +# ): +# number_of_asserted_projects += 1 +# assert len(project_invite_keys) == number_of_asserted_projects +# assert len(project_invite_keys) == len(invited_user.unit.projects) +# assert len(project_invite_keys) == 5 + +# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=token, +# json=new_unit_user, +# ) +# # No new mail should be sent for the token and neither for an invite +# assert mock_mail_send.call_count == 0 + +# assert response.status_code == http.HTTPStatus.BAD_REQUEST +# message = response.json.get("message") +# assert "user was already added to the system" in message + + +# def test_add_user_with_superadmin(client): +# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: +# token = tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client) +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=token, +# json=first_new_user, +# ) +# # One mail sent for partial token and one for the invite +# assert mock_mail_send.call_count == 2 + +# assert response.status_code == http.HTTPStatus.OK + +# invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() +# assert invited_user +# assert invited_user.email == first_new_user["email"] +# assert invited_user.role == first_new_user["role"] + +# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=token, +# json=first_new_user, +# ) +# # No new mail should be sent for the token and neither for an invite +# assert mock_mail_send.call_count == 0 + +# assert response.status_code == http.HTTPStatus.BAD_REQUEST +# message = response.json.get("message") +# assert "user was already added to the system" in message + + +# def test_add_user_existing_email_no_project(client): +# invited_user = models.Invite.query.filter_by( +# email=existing_invite["email"], role=existing_invite["role"] +# ).one_or_none() +# assert invited_user +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), +# json=existing_invite, +# ) +# assert response.status_code == http.HTTPStatus.BAD_REQUEST + + +# def test_add_unitadmin_user_with_unitpersonnel_permission_denied(client): +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# json=new_unit_admin, +# ) +# assert response.status_code == http.HTTPStatus.FORBIDDEN + +# invited_user = models.Invite.query.filter_by(email=new_unit_admin["email"]).one_or_none() +# assert invited_user is None + + +# # Add existing users to projects ################################# Add existing users to projects # +# def test_add_existing_user_without_project(client): +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# json=existing_research_user, +# ) +# assert response.status_code == http.HTTPStatus.BAD_REQUEST + + +# def test_research_user_cannot_add_existing_user_to_existing_project(client): +# user_copy = existing_research_user_to_existing_project.copy() +# project_id = user_copy.pop("project") + +# project = models.Project.query.filter_by(public_id=project_id).one_or_none() +# user = models.Email.query.filter_by( +# email=existing_research_user_to_existing_project["email"] +# ).one_or_none() +# project_user_before_addition = models.ProjectUsers.query.filter( +# sqlalchemy.and_( +# models.ProjectUsers.user_id == user.user_id, +# models.ProjectUsers.project_id == project.id, +# ) +# ).one_or_none() +# assert project_user_before_addition is None + +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["researchuser"]).token(client), +# query_string={"project": project_id}, +# json=user_copy, +# ) +# assert response.status_code == http.HTTPStatus.FORBIDDEN + +# project_user_after_addition = models.ProjectUsers.query.filter( +# sqlalchemy.and_( +# models.ProjectUsers.user_id == user.user_id, +# models.ProjectUsers.project_id == project.id, +# ) +# ).one_or_none() +# assert project_user_after_addition is None + + +# # projectowner adds researchuser2 to projects[0] +# def test_project_owner_can_add_existing_user_to_existing_project(client): +# user_copy = existing_research_user_to_existing_project.copy() +# project_id = user_copy.pop("project") + +# project = models.Project.query.filter_by(public_id=project_id).one_or_none() +# user = models.Email.query.filter_by( +# email=existing_research_user_to_existing_project["email"] +# ).one_or_none() +# project_user_before_addition = models.ProjectUsers.query.filter( +# sqlalchemy.and_( +# models.ProjectUsers.user_id == user.user_id, +# models.ProjectUsers.project_id == project.id, +# ) +# ).one_or_none() +# assert project_user_before_addition is None + +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["projectowner"]).token(client), +# query_string={"project": project_id}, +# json=user_copy, +# ) +# assert response.status_code == http.HTTPStatus.OK + +# project_user_after_addition = models.ProjectUsers.query.filter( +# sqlalchemy.and_( +# models.ProjectUsers.user_id == user.user_id, +# models.ProjectUsers.project_id == project.id, +# ) +# ).one_or_none() +# assert project_user_after_addition is not None + + +# def test_add_existing_user_to_existing_project(client): +# user_copy = existing_research_user_to_existing_project.copy() +# project_id = user_copy.pop("project") + +# project = models.Project.query.filter_by(public_id=project_id).one_or_none() +# user = models.Email.query.filter_by( +# email=existing_research_user_to_existing_project["email"] +# ).one_or_none() +# project_user_before_addition = models.ProjectUsers.query.filter( +# sqlalchemy.and_( +# models.ProjectUsers.user_id == user.user_id, +# models.ProjectUsers.project_id == project.id, +# ) +# ).one_or_none() +# assert project_user_before_addition is None + +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# query_string={"project": project_id}, +# json=user_copy, +# ) +# assert response.status_code == http.HTTPStatus.OK + +# project_user_after_addition = models.ProjectUsers.query.filter( +# sqlalchemy.and_( +# models.ProjectUsers.user_id == user.user_id, +# models.ProjectUsers.project_id == project.id, +# ) +# ).one_or_none() +# assert project_user_after_addition + + +# def test_add_existing_user_to_existing_project_no_mail_flag(client): +# "Test that an e-mail notification is not send when the --no-mail flag is used" + +# user_copy = existing_research_user_to_existing_project.copy() +# project_id = user_copy.pop("project") +# new_status = {"new_status": "Available"} +# user_copy["send_email"] = False +# token = tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client) + +# # make project available prior to test, otherwise an e-mail is never sent. +# response = client.post( +# tests.DDSEndpoint.PROJECT_STATUS, +# headers=token, +# query_string={"project": project_id}, +# data=json.dumps(new_status), +# content_type="application/json", +# ) + +# # Test mail sending is suppressed + +# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: +# with unittest.mock.patch.object( +# dds_web.api.user.AddUser, "compose_and_send_email_to_user" +# ) as mock_mail_func: +# print(user_copy) +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=token, +# query_string={"project": project_id}, +# data=json.dumps(user_copy), +# content_type="application/json", +# ) +# # assert that no mail is being sent. +# assert mock_mail_func.called == False +# assert mock_mail_send.call_count == 0 + +# assert response.status_code == http.HTTPStatus.OK +# assert "An e-mail notification has not been sent." in response.json["message"] + + +# def test_add_existing_user_to_existing_project_after_release(client): +# user_copy = existing_research_user_to_existing_project.copy() +# project_id = user_copy.pop("project") + +# project = models.Project.query.filter_by(public_id=project_id).one_or_none() +# user = models.Email.query.filter_by( +# email=existing_research_user_to_existing_project["email"] +# ).one_or_none() +# project_user_before_addition = models.ProjectUsers.query.filter( +# sqlalchemy.and_( +# models.ProjectUsers.user_id == user.user_id, +# models.ProjectUsers.project_id == project.id, +# ) +# ).one_or_none() +# assert project_user_before_addition is None + +# # release project +# response = client.post( +# tests.DDSEndpoint.PROJECT_STATUS, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# query_string={"project": project_id}, +# json={"new_status": "Available"}, +# ) +# assert response.status_code == http.HTTPStatus.OK +# assert project.current_status == "Available" + +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# query_string={"project": project_id}, +# json=user_copy, +# ) +# assert response.status_code == http.HTTPStatus.OK + +# project_user_after_addition = models.ProjectUsers.query.filter( +# sqlalchemy.and_( +# models.ProjectUsers.user_id == user.user_id, +# models.ProjectUsers.project_id == project.id, +# ) +# ).one_or_none() +# assert project_user_after_addition + + +# def test_add_existing_user_to_nonexistent_proj(client): +# user_copy = existing_research_user_to_nonexistent_proj.copy() +# project = user_copy.pop("project") +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# query_string={"project": project}, +# json=user_copy, +# ) +# assert response.status_code == http.HTTPStatus.BAD_REQUEST + + +# def test_existing_user_change_ownership(client): +# project = models.Project.query.filter_by( +# public_id=change_owner_existing_user["project"] +# ).one_or_none() +# user = models.Email.query.filter_by(email=change_owner_existing_user["email"]).one_or_none() +# project_user = models.ProjectUsers.query.filter( +# sqlalchemy.and_( +# models.ProjectUsers.user_id == user.user_id, +# models.ProjectUsers.project_id == project.id, +# ) +# ).one_or_none() + +# assert not project_user.owner + +# user_new_owner_status = change_owner_existing_user.copy() +# project = user_new_owner_status.pop("project") +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# query_string={"project": project}, +# json=user_new_owner_status, +# ) + +# assert response.status_code == http.HTTPStatus.OK + +# db.session.refresh(project_user) + +# assert project_user.owner + + +# def test_existing_user_change_ownership_same_permissions(client): +# user_same_ownership = submit_with_same_ownership.copy() +# project = user_same_ownership.pop("project") +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# query_string={"project": project}, +# json=user_same_ownership, +# ) +# assert response.status_code == http.HTTPStatus.FORBIDDEN + + +# def test_add_existing_user_with_unsuitable_role(client): +# user_with_unsuitable_role = existing_research_user_to_existing_project.copy() +# user_with_unsuitable_role["role"] = "Unit Admin" +# project = user_with_unsuitable_role.pop("project") +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# query_string={"project": project}, +# json=user_with_unsuitable_role, +# ) +# assert response.status_code == http.HTTPStatus.FORBIDDEN + + +# # Invite to project ########################################################### Invite to project # + + +# def test_invite_with_project_by_unituser(client): +# "Test that a new invite including a project can be created" + +# project = existing_project +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# query_string={"project": project}, +# json=first_new_user, +# ) +# assert response.status_code == http.HTTPStatus.OK + +# invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() +# assert invited_user +# assert invited_user.email == first_new_user["email"] +# assert invited_user.role == first_new_user["role"] + +# assert invited_user.nonce is not None +# assert invited_user.public_key is not None +# assert invited_user.private_key is not None + +# project_invite_keys = invited_user.project_invite_keys +# assert len(project_invite_keys) == 1 +# assert project_invite_keys[0].project.public_id == project +# assert not project_invite_keys[0].owner + + +# def test_add_project_to_existing_invite_by_unituser(client): +# "Test that a project can be associated with an existing invite" + +# # Create invite upfront + +# project = existing_project +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# json=first_new_user, +# ) +# assert response.status_code == http.HTTPStatus.OK + +# invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() + +# # Check that the invite has no projects yet + +# assert invited_user +# assert len(invited_user.project_invite_keys) == 0 + +# # Add project to existing invite + +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# query_string={"project": project}, +# json=first_new_user, +# ) + +# assert response.status_code == http.HTTPStatus.OK + +# # Check that the invite has now a project association +# project_invite_keys = invited_user.project_invite_keys +# assert len(project_invite_keys) == 1 +# assert project_invite_keys[0].project.public_id == project +# assert not project_invite_keys[0].owner - invited_user = models.Invite.query.filter_by( - email=first_new_user_invalid_email["email"] - ).one_or_none() - assert invited_user is None +# def test_update_project_to_existing_invite_by_unituser(client): +# "Test that project ownership can be updated for an existing invite" -def test_add_user_with_unitadmin(client): - with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: - token = tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client) - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=token, - json=first_new_user, - ) - # One mail sent for partial token and one for the invite - assert mock_mail_send.call_count == 2 - - assert response.status_code == http.HTTPStatus.OK - - invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() - assert invited_user - assert invited_user.email == first_new_user["email"] - assert invited_user.role == first_new_user["role"] +# # Create Invite upfront + +# project = existing_project +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# query_string={"project": project}, +# json=first_new_user, +# ) +# assert response.status_code == http.HTTPStatus.OK + +# project_obj = models.Project.query.filter_by(public_id=existing_project).one_or_none() +# invite_obj = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() - assert invited_user.nonce is not None - assert invited_user.public_key is not None - assert invited_user.private_key is not None - assert invited_user.project_invite_keys == [] +# project_invite = models.ProjectInviteKeys.query.filter( +# sqlalchemy.and_( +# models.ProjectInviteKeys.invite_id == invite_obj.id, +# models.ProjectUserKeys.project_id == project_obj.id, +# ) +# ).one_or_none() - # Repeating the invite should not send a new invite: - with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=token, - json=first_new_user, - ) - # No new mail should be sent for the token and neither for an invite - assert mock_mail_send.call_count == 0 - assert response.status_code == http.HTTPStatus.BAD_REQUEST - message = response.json.get("message") - assert "user was already added to the system" in message - - -def test_add_unit_user_with_unitadmin(client): - - with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: - token = tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client) - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=token, - json=new_unit_user, - ) - # One mail sent for partial token and one for the invite - assert mock_mail_send.call_count == 2 - - assert response.status_code == http.HTTPStatus.OK - - invited_user = models.Invite.query.filter_by(email=new_unit_user["email"]).one_or_none() - assert invited_user - assert invited_user.email == new_unit_user["email"] - assert invited_user.role == new_unit_user["role"] - - assert invited_user.nonce is not None - assert invited_user.public_key is not None - assert invited_user.private_key is not None - - project_invite_keys = invited_user.project_invite_keys - number_of_asserted_projects = 0 - for project_invite_key in project_invite_keys: - if ( - project_invite_key.project.public_id == "public_project_id" - or project_invite_key.project.public_id == "unused_project_id" - or project_invite_key.project.public_id == "restricted_project_id" - or project_invite_key.project.public_id == "second_public_project_id" - or project_invite_key.project.public_id == "file_testing_project" - ): - number_of_asserted_projects += 1 - assert len(project_invite_keys) == number_of_asserted_projects - assert len(project_invite_keys) == len(invited_user.unit.projects) - assert len(project_invite_keys) == 5 - - with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=token, - json=new_unit_user, - ) - # No new mail should be sent for the token and neither for an invite - assert mock_mail_send.call_count == 0 +# assert project_invite +# assert not project_invite.owner - assert response.status_code == http.HTTPStatus.BAD_REQUEST - message = response.json.get("message") - assert "user was already added to the system" in message - - -def test_add_user_with_superadmin(client): - with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: - token = tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client) - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=token, - json=first_new_user, - ) - # One mail sent for partial token and one for the invite - assert mock_mail_send.call_count == 2 - - assert response.status_code == http.HTTPStatus.OK - - invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() - assert invited_user - assert invited_user.email == first_new_user["email"] - assert invited_user.role == first_new_user["role"] - - with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=token, - json=first_new_user, - ) - # No new mail should be sent for the token and neither for an invite - assert mock_mail_send.call_count == 0 - - assert response.status_code == http.HTTPStatus.BAD_REQUEST - message = response.json.get("message") - assert "user was already added to the system" in message - - -def test_add_user_existing_email_no_project(client): - invited_user = models.Invite.query.filter_by( - email=existing_invite["email"], role=existing_invite["role"] - ).one_or_none() - assert invited_user - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), - json=existing_invite, - ) - assert response.status_code == http.HTTPStatus.BAD_REQUEST - - -def test_add_unitadmin_user_with_unitpersonnel_permission_denied(client): - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - json=new_unit_admin, - ) - assert response.status_code == http.HTTPStatus.FORBIDDEN - - invited_user = models.Invite.query.filter_by(email=new_unit_admin["email"]).one_or_none() - assert invited_user is None - - -# Add existing users to projects ################################# Add existing users to projects # -def test_add_existing_user_without_project(client): - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - json=existing_research_user, - ) - assert response.status_code == http.HTTPStatus.BAD_REQUEST - - -def test_research_user_cannot_add_existing_user_to_existing_project(client): - user_copy = existing_research_user_to_existing_project.copy() - project_id = user_copy.pop("project") - - project = models.Project.query.filter_by(public_id=project_id).one_or_none() - user = models.Email.query.filter_by( - email=existing_research_user_to_existing_project["email"] - ).one_or_none() - project_user_before_addition = models.ProjectUsers.query.filter( - sqlalchemy.and_( - models.ProjectUsers.user_id == user.user_id, - models.ProjectUsers.project_id == project.id, - ) - ).one_or_none() - assert project_user_before_addition is None - - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["researchuser"]).token(client), - query_string={"project": project_id}, - json=user_copy, - ) - assert response.status_code == http.HTTPStatus.FORBIDDEN - - project_user_after_addition = models.ProjectUsers.query.filter( - sqlalchemy.and_( - models.ProjectUsers.user_id == user.user_id, - models.ProjectUsers.project_id == project.id, - ) - ).one_or_none() - assert project_user_after_addition is None - - -# projectowner adds researchuser2 to projects[0] -def test_project_owner_can_add_existing_user_to_existing_project(client): - user_copy = existing_research_user_to_existing_project.copy() - project_id = user_copy.pop("project") - - project = models.Project.query.filter_by(public_id=project_id).one_or_none() - user = models.Email.query.filter_by( - email=existing_research_user_to_existing_project["email"] - ).one_or_none() - project_user_before_addition = models.ProjectUsers.query.filter( - sqlalchemy.and_( - models.ProjectUsers.user_id == user.user_id, - models.ProjectUsers.project_id == project.id, - ) - ).one_or_none() - assert project_user_before_addition is None - - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["projectowner"]).token(client), - query_string={"project": project_id}, - json=user_copy, - ) - assert response.status_code == http.HTTPStatus.OK - - project_user_after_addition = models.ProjectUsers.query.filter( - sqlalchemy.and_( - models.ProjectUsers.user_id == user.user_id, - models.ProjectUsers.project_id == project.id, - ) - ).one_or_none() - assert project_user_after_addition is not None - - -def test_add_existing_user_to_existing_project(client): - user_copy = existing_research_user_to_existing_project.copy() - project_id = user_copy.pop("project") - - project = models.Project.query.filter_by(public_id=project_id).one_or_none() - user = models.Email.query.filter_by( - email=existing_research_user_to_existing_project["email"] - ).one_or_none() - project_user_before_addition = models.ProjectUsers.query.filter( - sqlalchemy.and_( - models.ProjectUsers.user_id == user.user_id, - models.ProjectUsers.project_id == project.id, - ) - ).one_or_none() - assert project_user_before_addition is None - - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - query_string={"project": project_id}, - json=user_copy, - ) - assert response.status_code == http.HTTPStatus.OK - - project_user_after_addition = models.ProjectUsers.query.filter( - sqlalchemy.and_( - models.ProjectUsers.user_id == user.user_id, - models.ProjectUsers.project_id == project.id, - ) - ).one_or_none() - assert project_user_after_addition - - -def test_add_existing_user_to_existing_project_no_mail_flag(client): - "Test that an e-mail notification is not send when the --no-mail flag is used" - - user_copy = existing_research_user_to_existing_project.copy() - project_id = user_copy.pop("project") - new_status = {"new_status": "Available"} - user_copy["send_email"] = False - token = tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client) - - # make project available prior to test, otherwise an e-mail is never sent. - response = client.post( - tests.DDSEndpoint.PROJECT_STATUS, - headers=token, - query_string={"project": project_id}, - data=json.dumps(new_status), - content_type="application/json", - ) - - # Test mail sending is suppressed - - with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: - with unittest.mock.patch.object( - dds_web.api.user.AddUser, "compose_and_send_email_to_user" - ) as mock_mail_func: - print(user_copy) - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=token, - query_string={"project": project_id}, - data=json.dumps(user_copy), - content_type="application/json", - ) - # assert that no mail is being sent. - assert mock_mail_func.called == False - assert mock_mail_send.call_count == 0 - - assert response.status_code == http.HTTPStatus.OK - assert "An e-mail notification has not been sent." in response.json["message"] - - -def test_add_existing_user_to_existing_project_after_release(client): - user_copy = existing_research_user_to_existing_project.copy() - project_id = user_copy.pop("project") - - project = models.Project.query.filter_by(public_id=project_id).one_or_none() - user = models.Email.query.filter_by( - email=existing_research_user_to_existing_project["email"] - ).one_or_none() - project_user_before_addition = models.ProjectUsers.query.filter( - sqlalchemy.and_( - models.ProjectUsers.user_id == user.user_id, - models.ProjectUsers.project_id == project.id, - ) - ).one_or_none() - assert project_user_before_addition is None - - # release project - response = client.post( - tests.DDSEndpoint.PROJECT_STATUS, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - query_string={"project": project_id}, - json={"new_status": "Available"}, - ) - assert response.status_code == http.HTTPStatus.OK - assert project.current_status == "Available" - - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - query_string={"project": project_id}, - json=user_copy, - ) - assert response.status_code == http.HTTPStatus.OK - - project_user_after_addition = models.ProjectUsers.query.filter( - sqlalchemy.and_( - models.ProjectUsers.user_id == user.user_id, - models.ProjectUsers.project_id == project.id, - ) - ).one_or_none() - assert project_user_after_addition - - -def test_add_existing_user_to_nonexistent_proj(client): - user_copy = existing_research_user_to_nonexistent_proj.copy() - project = user_copy.pop("project") - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - query_string={"project": project}, - json=user_copy, - ) - assert response.status_code == http.HTTPStatus.BAD_REQUEST - - -def test_existing_user_change_ownership(client): - project = models.Project.query.filter_by( - public_id=change_owner_existing_user["project"] - ).one_or_none() - user = models.Email.query.filter_by(email=change_owner_existing_user["email"]).one_or_none() - project_user = models.ProjectUsers.query.filter( - sqlalchemy.and_( - models.ProjectUsers.user_id == user.user_id, - models.ProjectUsers.project_id == project.id, - ) - ).one_or_none() +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# query_string={"project": project}, +# json=first_new_owner, +# ) - assert not project_user.owner +# assert response.status_code == http.HTTPStatus.OK + +# db.session.refresh(project_invite) - user_new_owner_status = change_owner_existing_user.copy() - project = user_new_owner_status.pop("project") - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - query_string={"project": project}, - json=user_new_owner_status, - ) +# assert project_invite.owner + + +# def test_invited_as_owner_and_researcher_to_different_project(client): +# "Test that an invite can be owner of one project and researcher of another" - assert response.status_code == http.HTTPStatus.OK +# # Create Invite upfront as owner - db.session.refresh(project_user) +# project = existing_project +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# query_string={"project": project}, +# json=first_new_owner, +# ) +# assert response.status_code == http.HTTPStatus.OK - assert project_user.owner +# # Perform second invite as researcher +# project2 = existing_project_2 +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# query_string={"project": project2}, +# json=first_new_user, +# ) +# assert response.status_code == http.HTTPStatus.OK +# project_obj_owner = models.Project.query.filter_by(public_id=existing_project).one_or_none() +# project_obj_not_owner = models.Project.query.filter_by( +# public_id=existing_project_2 +# ).one_or_none() -def test_existing_user_change_ownership_same_permissions(client): - user_same_ownership = submit_with_same_ownership.copy() - project = user_same_ownership.pop("project") - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - query_string={"project": project}, - json=user_same_ownership, - ) - assert response.status_code == http.HTTPStatus.FORBIDDEN +# invite_obj = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() +# project_invite_owner = models.ProjectInviteKeys.query.filter( +# sqlalchemy.and_( +# models.ProjectInviteKeys.invite_id == invite_obj.id, +# models.ProjectInviteKeys.project_id == project_obj_owner.id, +# ) +# ).one_or_none() -def test_add_existing_user_with_unsuitable_role(client): - user_with_unsuitable_role = existing_research_user_to_existing_project.copy() - user_with_unsuitable_role["role"] = "Unit Admin" - project = user_with_unsuitable_role.pop("project") - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - query_string={"project": project}, - json=user_with_unsuitable_role, - ) - assert response.status_code == http.HTTPStatus.FORBIDDEN +# assert project_invite_owner +# assert project_invite_owner.owner +# project_invite_not_owner = models.ProjectInviteKeys.query.filter( +# sqlalchemy.and_( +# models.ProjectInviteKeys.invite_id == invite_obj.id, +# models.ProjectInviteKeys.project_id == project_obj_not_owner.id, +# ) +# ).one_or_none() + +# assert project_invite_not_owner +# assert not project_invite_not_owner.owner + +# # Owner or not should not be stored on the invite +# assert invite_obj.role == "Researcher" + + +# def test_invite_to_project_by_project_owner(client): +# "Test that a project owner can invite to its project" -# Invite to project ########################################################### Invite to project # +# project = existing_project +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["projectowner"]).token(client), +# query_string={"project": project}, +# json=first_new_user, +# ) +# assert response.status_code == http.HTTPStatus.OK + +# invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() +# assert invited_user +# assert invited_user.email == first_new_user["email"] +# assert invited_user.role == first_new_user["role"] + +# assert invited_user.nonce is not None +# assert invited_user.public_key is not None +# assert invited_user.private_key is not None + +# project_invite_keys = invited_user.project_invite_keys +# assert len(project_invite_keys) == 1 +# assert project_invite_keys[0].project.public_id == project +# assert not project_invite_keys[0].owner -def test_invite_with_project_by_unituser(client): - "Test that a new invite including a project can be created" - - project = existing_project - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - query_string={"project": project}, - json=first_new_user, - ) - assert response.status_code == http.HTTPStatus.OK - - invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() - assert invited_user - assert invited_user.email == first_new_user["email"] - assert invited_user.role == first_new_user["role"] - - assert invited_user.nonce is not None - assert invited_user.public_key is not None - assert invited_user.private_key is not None - - project_invite_keys = invited_user.project_invite_keys - assert len(project_invite_keys) == 1 - assert project_invite_keys[0].project.public_id == project - assert not project_invite_keys[0].owner - - -def test_add_project_to_existing_invite_by_unituser(client): - "Test that a project can be associated with an existing invite" - - # Create invite upfront - - project = existing_project - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - json=first_new_user, - ) - assert response.status_code == http.HTTPStatus.OK - - invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() - - # Check that the invite has no projects yet - - assert invited_user - assert len(invited_user.project_invite_keys) == 0 - - # Add project to existing invite - - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - query_string={"project": project}, - json=first_new_user, - ) - - assert response.status_code == http.HTTPStatus.OK - - # Check that the invite has now a project association - project_invite_keys = invited_user.project_invite_keys - assert len(project_invite_keys) == 1 - assert project_invite_keys[0].project.public_id == project - assert not project_invite_keys[0].owner - - -def test_update_project_to_existing_invite_by_unituser(client): - "Test that project ownership can be updated for an existing invite" - - # Create Invite upfront - +def test_add_researcher_and_owner_to_project_with_superadmin(client): project = existing_project - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - query_string={"project": project}, - json=first_new_user, - ) - assert response.status_code == http.HTTPStatus.OK - - project_obj = models.Project.query.filter_by(public_id=existing_project).one_or_none() - invite_obj = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() - - project_invite = models.ProjectInviteKeys.query.filter( - sqlalchemy.and_( - models.ProjectInviteKeys.invite_id == invite_obj.id, - models.ProjectUserKeys.project_id == project_obj.id, + for x in [first_new_user, first_new_owner, new_unit_user, new_unit_admin]: + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client), + query_string={"project": project}, + json=x, ) - ).one_or_none() - - assert project_invite - assert not project_invite.owner - - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - query_string={"project": project}, - json=first_new_owner, - ) - - assert response.status_code == http.HTTPStatus.OK - - db.session.refresh(project_invite) - - assert project_invite.owner - + assert response.status_code == http.HTTPStatus.BAD_REQUEST -def test_invited_as_owner_and_researcher_to_different_project(client): - "Test that an invite can be owner of one project and researcher of another" + invited_user = models.Invite.query.filter_by(email=x["email"]).one_or_none() + assert not invited_user - # Create Invite upfront as owner +def test_add_unituser_and_admin_no_unit_with_superadmin(client): project = existing_project - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - query_string={"project": project}, - json=first_new_owner, - ) - assert response.status_code == http.HTTPStatus.OK - - # Perform second invite as researcher - project2 = existing_project_2 - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - query_string={"project": project2}, - json=first_new_user, - ) - assert response.status_code == http.HTTPStatus.OK - - project_obj_owner = models.Project.query.filter_by(public_id=existing_project).one_or_none() - project_obj_not_owner = models.Project.query.filter_by( - public_id=existing_project_2 - ).one_or_none() - - invite_obj = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() - - project_invite_owner = models.ProjectInviteKeys.query.filter( - sqlalchemy.and_( - models.ProjectInviteKeys.invite_id == invite_obj.id, - models.ProjectInviteKeys.project_id == project_obj_owner.id, - ) - ).one_or_none() - - assert project_invite_owner - assert project_invite_owner.owner - - project_invite_not_owner = models.ProjectInviteKeys.query.filter( - sqlalchemy.and_( - models.ProjectInviteKeys.invite_id == invite_obj.id, - models.ProjectInviteKeys.project_id == project_obj_not_owner.id, + for x in [new_unit_user, new_unit_admin]: + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client), + json=x, ) - ).one_or_none() + # assert response.status_code == http.HTTPStatus.BAD_REQUEST - assert project_invite_not_owner - assert not project_invite_not_owner.owner - - # Owner or not should not be stored on the invite - assert invite_obj.role == "Researcher" - - -def test_invite_to_project_by_project_owner(client): - "Test that a project owner can invite to its project" - - project = existing_project - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["projectowner"]).token(client), - query_string={"project": project}, - json=first_new_user, - ) - assert response.status_code == http.HTTPStatus.OK - - invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() - assert invited_user - assert invited_user.email == first_new_user["email"] - assert invited_user.role == first_new_user["role"] - - assert invited_user.nonce is not None - assert invited_user.public_key is not None - assert invited_user.private_key is not None - - project_invite_keys = invited_user.project_invite_keys - assert len(project_invite_keys) == 1 - assert project_invite_keys[0].project.public_id == project - assert not project_invite_keys[0].owner + assert "You need to specify a unit" in response.json["message"] + invited_user = models.Invite.query.filter_by(email=x["email"]).one_or_none() + assert not invited_user From 1c19d51f28768a9d7ce1dbe03815033638cea4e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Fri, 11 Mar 2022 15:42:26 +0100 Subject: [PATCH 043/293] uncommented http status --- tests/test_user_add.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_user_add.py b/tests/test_user_add.py index f7deb4714..810a185a7 100644 --- a/tests/test_user_add.py +++ b/tests/test_user_add.py @@ -729,8 +729,8 @@ def test_add_unituser_and_admin_no_unit_with_superadmin(client): headers=tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client), json=x, ) - # assert response.status_code == http.HTTPStatus.BAD_REQUEST assert "You need to specify a unit" in response.json["message"] + assert response.status_code == http.HTTPStatus.BAD_REQUEST invited_user = models.Invite.query.filter_by(email=x["email"]).one_or_none() assert not invited_user From 6ff67655b3fb68f49b8cbe842621497793f097e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Sat, 12 Mar 2022 16:58:26 +0100 Subject: [PATCH 044/293] tests --- dds_web/api/user.py | 2 ++ tests/test_user_add.py | 82 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index 10ad15042..0a63d7bdf 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -6,6 +6,7 @@ # Standard library import os +from pyexpat import model from re import T import smtplib import time @@ -263,6 +264,7 @@ def invite_user(email, new_user_role, project=None, unit=None): try: db.session.commit() except sqlalchemy.exc.SQLAlchemyError as sqlerr: + db.session.rollback() raise ddserr.DatabaseError(message=str(sqlerr)) AddUser.compose_and_send_email_to_user( diff --git a/tests/test_user_add.py b/tests/test_user_add.py index 810a185a7..6d03cf294 100644 --- a/tests/test_user_add.py +++ b/tests/test_user_add.py @@ -21,7 +21,7 @@ first_new_user_invalid_role = {**first_new_email, "role": "Invalid Role"} first_new_user_invalid_email = {"email": "first_invalid_email", "role": first_new_user["role"]} existing_invite = {"email": "existing_invite_email@mailtrap.io", "role": "Researcher"} -new_unit_admin = {"email": "new_unit_admin@mailtrap.io", "role": "Super Admin"} +new_unit_admin = {"email": "new_unit_admin@mailtrap.io", "role": "Unit Admin"} new_unit_user = {"email": "new_unit_user@mailtrap.io", "role": "Unit Personnel"} existing_research_user = {"email": "researchuser2@mailtrap.io", "role": "Researcher"} existing_research_user_owner = {"email": "researchuser2@mailtrap.io", "role": "Project Owner"} @@ -734,3 +734,83 @@ def test_add_unituser_and_admin_no_unit_with_superadmin(client): assert response.status_code == http.HTTPStatus.BAD_REQUEST invited_user = models.Invite.query.filter_by(email=x["email"]).one_or_none() assert not invited_user + + +def test_add_researchuser_project_no_access_unit_admin(client): + project = models.Project.query.filter_by(public_id=existing_project).one_or_none() + assert project + + project_user_key = models.ProjectUserKeys.query.filter_by( + user_id="unitadmin", project_id=project.id + ).one_or_none() + assert project_user_key + + db.session.delete(project_user_key) + project_user_key = models.ProjectUserKeys.query.filter_by( + user_id="unitadmin", project_id=project.id + ).one_or_none() + assert not project_user_key + + for x in [first_new_user, first_new_owner]: + + invited_user = models.Invite.query.filter_by(email=x["email"]).one_or_none() + print(invited_user) + assert not invited_user + + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), + json=x, + query_string={"project": project.public_id}, + ) + + assert response.status_code == http.HTTPStatus.BAD_REQUEST + assert "The user could not be added to the project(s)" in response.json["message"] + + assert "errors" in response.json + assert project.public_id in response.json["errors"] + assert ( + "You do not have access to the specified project." + in response.json["errors"][project.public_id] + ) + + invited_user = models.Invite.query.filter_by(email=x["email"]).one_or_none() + assert invited_user + + +# def test_add_unituser_project_no_access_unit_admin(client): +# project = models.Project.query.filter_by(public_id=existing_project).one_or_none() +# assert project + +# project_user_key = models.ProjectUserKeys.query.filter_by( +# user_id="unitadmin", project_id=project.id +# ).one_or_none() +# assert project_user_key + +# db.session.delete(project_user_key) +# project_user_key = models.ProjectUserKeys.query.filter_by( +# user_id="unitadmin", project_id=project.id +# ).one_or_none() +# assert not project_user_key + +# for x in [first_new_user, first_new_owner]: + +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), +# json=x, +# query_string={"project": project.public_id}, +# ) + +# assert response.status_code == http.HTTPStatus.BAD_REQUEST +# assert "The user could not be added to the project(s)" in response.json["message"] + +# assert "errors" in response.json +# assert project.public_id in response.json["errors"] +# assert ( +# "You do not have access to the specified project." +# in response.json["errors"][project.public_id] +# ) + +# invited_user = models.Invite.query.filter_by(email=x["email"]).one_or_none() +# assert not invited_user From 46ab67ff2358bfad911cff4e3cb571206f91e929 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Sun, 13 Mar 2022 11:20:23 +0100 Subject: [PATCH 045/293] added tests for user.py --- tests/test_user_add.py | 1431 ++++++++++++++++++++-------------------- 1 file changed, 717 insertions(+), 714 deletions(-) diff --git a/tests/test_user_add.py b/tests/test_user_add.py index 6d03cf294..46f4fd580 100644 --- a/tests/test_user_add.py +++ b/tests/test_user_add.py @@ -44,669 +44,670 @@ } # Inviting Users ################################################################# Inviting Users # -# def test_add_user_with_researcher(client): -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["researchuser"]).token(client), -# json=first_new_user, -# ) -# assert response.status_code == http.HTTPStatus.FORBIDDEN -# invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() -# assert invited_user is None - - -# def test_add_user_with_unituser_no_role(client): -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), -# json=first_new_email, -# ) -# assert response.status_code == http.HTTPStatus.BAD_REQUEST -# invited_user = models.Invite.query.filter_by(email=first_new_email["email"]).one_or_none() -# assert invited_user is None - - -# def test_add_user_with_unitadmin_with_extraargs(client): -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), -# json=first_new_user_extra_args, -# ) -# assert response.status_code == http.HTTPStatus.OK -# invited_user = models.Invite.query.filter_by( -# email=first_new_user_extra_args["email"] -# ).one_or_none() -# assert invited_user - - -# def test_add_user_with_unitadmin_and_invalid_role(client): -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), -# json=first_new_user_invalid_role, -# ) -# assert response.status_code == http.HTTPStatus.BAD_REQUEST -# invited_user = models.Invite.query.filter_by( -# email=first_new_user_invalid_role["email"] -# ).one_or_none() -# assert invited_user is None - - -# def test_add_user_with_unitadmin_and_invalid_email(client): -# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), -# json=first_new_user_invalid_email, -# ) -# assert response.status_code == http.HTTPStatus.BAD_REQUEST -# # An email is always sent when receiving the partial token -# mock_mail_send.assert_called_once() - -# invited_user = models.Invite.query.filter_by( -# email=first_new_user_invalid_email["email"] -# ).one_or_none() -# assert invited_user is None - - -# def test_add_user_with_unitadmin(client): -# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: -# token = tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client) -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=token, -# json=first_new_user, -# ) -# # One mail sent for partial token and one for the invite -# assert mock_mail_send.call_count == 2 - -# assert response.status_code == http.HTTPStatus.OK - -# invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() -# assert invited_user -# assert invited_user.email == first_new_user["email"] -# assert invited_user.role == first_new_user["role"] - -# assert invited_user.nonce is not None -# assert invited_user.public_key is not None -# assert invited_user.private_key is not None -# assert invited_user.project_invite_keys == [] - -# # Repeating the invite should not send a new invite: -# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=token, -# json=first_new_user, -# ) -# # No new mail should be sent for the token and neither for an invite -# assert mock_mail_send.call_count == 0 -# assert response.status_code == http.HTTPStatus.BAD_REQUEST -# message = response.json.get("message") -# assert "user was already added to the system" in message - - -# def test_add_unit_user_with_unitadmin(client): - -# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: -# token = tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client) -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=token, -# json=new_unit_user, -# ) -# # One mail sent for partial token and one for the invite -# assert mock_mail_send.call_count == 2 - -# assert response.status_code == http.HTTPStatus.OK - -# invited_user = models.Invite.query.filter_by(email=new_unit_user["email"]).one_or_none() -# assert invited_user -# assert invited_user.email == new_unit_user["email"] -# assert invited_user.role == new_unit_user["role"] - -# assert invited_user.nonce is not None -# assert invited_user.public_key is not None -# assert invited_user.private_key is not None - -# project_invite_keys = invited_user.project_invite_keys -# number_of_asserted_projects = 0 -# for project_invite_key in project_invite_keys: -# if ( -# project_invite_key.project.public_id == "public_project_id" -# or project_invite_key.project.public_id == "unused_project_id" -# or project_invite_key.project.public_id == "restricted_project_id" -# or project_invite_key.project.public_id == "second_public_project_id" -# or project_invite_key.project.public_id == "file_testing_project" -# ): -# number_of_asserted_projects += 1 -# assert len(project_invite_keys) == number_of_asserted_projects -# assert len(project_invite_keys) == len(invited_user.unit.projects) -# assert len(project_invite_keys) == 5 - -# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=token, -# json=new_unit_user, -# ) -# # No new mail should be sent for the token and neither for an invite -# assert mock_mail_send.call_count == 0 - -# assert response.status_code == http.HTTPStatus.BAD_REQUEST -# message = response.json.get("message") -# assert "user was already added to the system" in message - - -# def test_add_user_with_superadmin(client): -# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: -# token = tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client) -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=token, -# json=first_new_user, -# ) -# # One mail sent for partial token and one for the invite -# assert mock_mail_send.call_count == 2 - -# assert response.status_code == http.HTTPStatus.OK - -# invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() -# assert invited_user -# assert invited_user.email == first_new_user["email"] -# assert invited_user.role == first_new_user["role"] - -# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=token, -# json=first_new_user, -# ) -# # No new mail should be sent for the token and neither for an invite -# assert mock_mail_send.call_count == 0 - -# assert response.status_code == http.HTTPStatus.BAD_REQUEST -# message = response.json.get("message") -# assert "user was already added to the system" in message - - -# def test_add_user_existing_email_no_project(client): -# invited_user = models.Invite.query.filter_by( -# email=existing_invite["email"], role=existing_invite["role"] -# ).one_or_none() -# assert invited_user -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), -# json=existing_invite, -# ) -# assert response.status_code == http.HTTPStatus.BAD_REQUEST - - -# def test_add_unitadmin_user_with_unitpersonnel_permission_denied(client): -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# json=new_unit_admin, -# ) -# assert response.status_code == http.HTTPStatus.FORBIDDEN - -# invited_user = models.Invite.query.filter_by(email=new_unit_admin["email"]).one_or_none() -# assert invited_user is None - - -# # Add existing users to projects ################################# Add existing users to projects # -# def test_add_existing_user_without_project(client): -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# json=existing_research_user, -# ) -# assert response.status_code == http.HTTPStatus.BAD_REQUEST - - -# def test_research_user_cannot_add_existing_user_to_existing_project(client): -# user_copy = existing_research_user_to_existing_project.copy() -# project_id = user_copy.pop("project") - -# project = models.Project.query.filter_by(public_id=project_id).one_or_none() -# user = models.Email.query.filter_by( -# email=existing_research_user_to_existing_project["email"] -# ).one_or_none() -# project_user_before_addition = models.ProjectUsers.query.filter( -# sqlalchemy.and_( -# models.ProjectUsers.user_id == user.user_id, -# models.ProjectUsers.project_id == project.id, -# ) -# ).one_or_none() -# assert project_user_before_addition is None - -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["researchuser"]).token(client), -# query_string={"project": project_id}, -# json=user_copy, -# ) -# assert response.status_code == http.HTTPStatus.FORBIDDEN - -# project_user_after_addition = models.ProjectUsers.query.filter( -# sqlalchemy.and_( -# models.ProjectUsers.user_id == user.user_id, -# models.ProjectUsers.project_id == project.id, -# ) -# ).one_or_none() -# assert project_user_after_addition is None - - -# # projectowner adds researchuser2 to projects[0] -# def test_project_owner_can_add_existing_user_to_existing_project(client): -# user_copy = existing_research_user_to_existing_project.copy() -# project_id = user_copy.pop("project") - -# project = models.Project.query.filter_by(public_id=project_id).one_or_none() -# user = models.Email.query.filter_by( -# email=existing_research_user_to_existing_project["email"] -# ).one_or_none() -# project_user_before_addition = models.ProjectUsers.query.filter( -# sqlalchemy.and_( -# models.ProjectUsers.user_id == user.user_id, -# models.ProjectUsers.project_id == project.id, -# ) -# ).one_or_none() -# assert project_user_before_addition is None - -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["projectowner"]).token(client), -# query_string={"project": project_id}, -# json=user_copy, -# ) -# assert response.status_code == http.HTTPStatus.OK - -# project_user_after_addition = models.ProjectUsers.query.filter( -# sqlalchemy.and_( -# models.ProjectUsers.user_id == user.user_id, -# models.ProjectUsers.project_id == project.id, -# ) -# ).one_or_none() -# assert project_user_after_addition is not None - - -# def test_add_existing_user_to_existing_project(client): -# user_copy = existing_research_user_to_existing_project.copy() -# project_id = user_copy.pop("project") - -# project = models.Project.query.filter_by(public_id=project_id).one_or_none() -# user = models.Email.query.filter_by( -# email=existing_research_user_to_existing_project["email"] -# ).one_or_none() -# project_user_before_addition = models.ProjectUsers.query.filter( -# sqlalchemy.and_( -# models.ProjectUsers.user_id == user.user_id, -# models.ProjectUsers.project_id == project.id, -# ) -# ).one_or_none() -# assert project_user_before_addition is None - -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# query_string={"project": project_id}, -# json=user_copy, -# ) -# assert response.status_code == http.HTTPStatus.OK - -# project_user_after_addition = models.ProjectUsers.query.filter( -# sqlalchemy.and_( -# models.ProjectUsers.user_id == user.user_id, -# models.ProjectUsers.project_id == project.id, -# ) -# ).one_or_none() -# assert project_user_after_addition - - -# def test_add_existing_user_to_existing_project_no_mail_flag(client): -# "Test that an e-mail notification is not send when the --no-mail flag is used" - -# user_copy = existing_research_user_to_existing_project.copy() -# project_id = user_copy.pop("project") -# new_status = {"new_status": "Available"} -# user_copy["send_email"] = False -# token = tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client) - -# # make project available prior to test, otherwise an e-mail is never sent. -# response = client.post( -# tests.DDSEndpoint.PROJECT_STATUS, -# headers=token, -# query_string={"project": project_id}, -# data=json.dumps(new_status), -# content_type="application/json", -# ) - -# # Test mail sending is suppressed - -# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: -# with unittest.mock.patch.object( -# dds_web.api.user.AddUser, "compose_and_send_email_to_user" -# ) as mock_mail_func: -# print(user_copy) -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=token, -# query_string={"project": project_id}, -# data=json.dumps(user_copy), -# content_type="application/json", -# ) -# # assert that no mail is being sent. -# assert mock_mail_func.called == False -# assert mock_mail_send.call_count == 0 - -# assert response.status_code == http.HTTPStatus.OK -# assert "An e-mail notification has not been sent." in response.json["message"] - - -# def test_add_existing_user_to_existing_project_after_release(client): -# user_copy = existing_research_user_to_existing_project.copy() -# project_id = user_copy.pop("project") - -# project = models.Project.query.filter_by(public_id=project_id).one_or_none() -# user = models.Email.query.filter_by( -# email=existing_research_user_to_existing_project["email"] -# ).one_or_none() -# project_user_before_addition = models.ProjectUsers.query.filter( -# sqlalchemy.and_( -# models.ProjectUsers.user_id == user.user_id, -# models.ProjectUsers.project_id == project.id, -# ) -# ).one_or_none() -# assert project_user_before_addition is None - -# # release project -# response = client.post( -# tests.DDSEndpoint.PROJECT_STATUS, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# query_string={"project": project_id}, -# json={"new_status": "Available"}, -# ) -# assert response.status_code == http.HTTPStatus.OK -# assert project.current_status == "Available" - -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# query_string={"project": project_id}, -# json=user_copy, -# ) -# assert response.status_code == http.HTTPStatus.OK - -# project_user_after_addition = models.ProjectUsers.query.filter( -# sqlalchemy.and_( -# models.ProjectUsers.user_id == user.user_id, -# models.ProjectUsers.project_id == project.id, -# ) -# ).one_or_none() -# assert project_user_after_addition - - -# def test_add_existing_user_to_nonexistent_proj(client): -# user_copy = existing_research_user_to_nonexistent_proj.copy() -# project = user_copy.pop("project") -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# query_string={"project": project}, -# json=user_copy, -# ) -# assert response.status_code == http.HTTPStatus.BAD_REQUEST - - -# def test_existing_user_change_ownership(client): -# project = models.Project.query.filter_by( -# public_id=change_owner_existing_user["project"] -# ).one_or_none() -# user = models.Email.query.filter_by(email=change_owner_existing_user["email"]).one_or_none() -# project_user = models.ProjectUsers.query.filter( -# sqlalchemy.and_( -# models.ProjectUsers.user_id == user.user_id, -# models.ProjectUsers.project_id == project.id, -# ) -# ).one_or_none() - -# assert not project_user.owner - -# user_new_owner_status = change_owner_existing_user.copy() -# project = user_new_owner_status.pop("project") -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# query_string={"project": project}, -# json=user_new_owner_status, -# ) - -# assert response.status_code == http.HTTPStatus.OK - -# db.session.refresh(project_user) - -# assert project_user.owner - - -# def test_existing_user_change_ownership_same_permissions(client): -# user_same_ownership = submit_with_same_ownership.copy() -# project = user_same_ownership.pop("project") -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# query_string={"project": project}, -# json=user_same_ownership, -# ) -# assert response.status_code == http.HTTPStatus.FORBIDDEN - - -# def test_add_existing_user_with_unsuitable_role(client): -# user_with_unsuitable_role = existing_research_user_to_existing_project.copy() -# user_with_unsuitable_role["role"] = "Unit Admin" -# project = user_with_unsuitable_role.pop("project") -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# query_string={"project": project}, -# json=user_with_unsuitable_role, -# ) -# assert response.status_code == http.HTTPStatus.FORBIDDEN - - -# # Invite to project ########################################################### Invite to project # - - -# def test_invite_with_project_by_unituser(client): -# "Test that a new invite including a project can be created" - -# project = existing_project -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# query_string={"project": project}, -# json=first_new_user, -# ) -# assert response.status_code == http.HTTPStatus.OK - -# invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() -# assert invited_user -# assert invited_user.email == first_new_user["email"] -# assert invited_user.role == first_new_user["role"] - -# assert invited_user.nonce is not None -# assert invited_user.public_key is not None -# assert invited_user.private_key is not None - -# project_invite_keys = invited_user.project_invite_keys -# assert len(project_invite_keys) == 1 -# assert project_invite_keys[0].project.public_id == project -# assert not project_invite_keys[0].owner - - -# def test_add_project_to_existing_invite_by_unituser(client): -# "Test that a project can be associated with an existing invite" - -# # Create invite upfront - -# project = existing_project -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# json=first_new_user, -# ) -# assert response.status_code == http.HTTPStatus.OK - -# invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() - -# # Check that the invite has no projects yet - -# assert invited_user -# assert len(invited_user.project_invite_keys) == 0 - -# # Add project to existing invite - -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# query_string={"project": project}, -# json=first_new_user, -# ) - -# assert response.status_code == http.HTTPStatus.OK - -# # Check that the invite has now a project association -# project_invite_keys = invited_user.project_invite_keys -# assert len(project_invite_keys) == 1 -# assert project_invite_keys[0].project.public_id == project -# assert not project_invite_keys[0].owner +def test_add_user_with_researcher(client): + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["researchuser"]).token(client), + json=first_new_user, + ) + assert response.status_code == http.HTTPStatus.FORBIDDEN + invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() + assert invited_user is None + + +def test_add_user_with_unituser_no_role(client): + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), + json=first_new_email, + ) + assert response.status_code == http.HTTPStatus.BAD_REQUEST + invited_user = models.Invite.query.filter_by(email=first_new_email["email"]).one_or_none() + assert invited_user is None + + +def test_add_user_with_unitadmin_with_extraargs(client): + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), + json=first_new_user_extra_args, + ) + assert response.status_code == http.HTTPStatus.OK + invited_user = models.Invite.query.filter_by( + email=first_new_user_extra_args["email"] + ).one_or_none() + assert invited_user + + +def test_add_user_with_unitadmin_and_invalid_role(client): + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), + json=first_new_user_invalid_role, + ) + assert response.status_code == http.HTTPStatus.BAD_REQUEST + invited_user = models.Invite.query.filter_by( + email=first_new_user_invalid_role["email"] + ).one_or_none() + assert invited_user is None + + +def test_add_user_with_unitadmin_and_invalid_email(client): + with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), + json=first_new_user_invalid_email, + ) + assert response.status_code == http.HTTPStatus.BAD_REQUEST + # An email is always sent when receiving the partial token + mock_mail_send.assert_called_once() + + invited_user = models.Invite.query.filter_by( + email=first_new_user_invalid_email["email"] + ).one_or_none() + assert invited_user is None + + +def test_add_user_with_unitadmin(client): + with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: + token = tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client) + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=token, + json=first_new_user, + ) + # One mail sent for partial token and one for the invite + assert mock_mail_send.call_count == 2 + + assert response.status_code == http.HTTPStatus.OK + + invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() + assert invited_user + assert invited_user.email == first_new_user["email"] + assert invited_user.role == first_new_user["role"] + + assert invited_user.nonce is not None + assert invited_user.public_key is not None + assert invited_user.private_key is not None + assert invited_user.project_invite_keys == [] + + # Repeating the invite should not send a new invite: + with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=token, + json=first_new_user, + ) + # No new mail should be sent for the token and neither for an invite + assert mock_mail_send.call_count == 0 + assert response.status_code == http.HTTPStatus.BAD_REQUEST + message = response.json.get("message") + assert "user was already added to the system" in message + + +def test_add_unit_user_with_unitadmin(client): + + with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: + token = tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client) + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=token, + json=new_unit_user, + ) + # One mail sent for partial token and one for the invite + assert mock_mail_send.call_count == 2 + + assert response.status_code == http.HTTPStatus.OK + + invited_user = models.Invite.query.filter_by(email=new_unit_user["email"]).one_or_none() + assert invited_user + assert invited_user.email == new_unit_user["email"] + assert invited_user.role == new_unit_user["role"] + + assert invited_user.nonce is not None + assert invited_user.public_key is not None + assert invited_user.private_key is not None + + project_invite_keys = invited_user.project_invite_keys + number_of_asserted_projects = 0 + for project_invite_key in project_invite_keys: + if ( + project_invite_key.project.public_id == "public_project_id" + or project_invite_key.project.public_id == "unused_project_id" + or project_invite_key.project.public_id == "restricted_project_id" + or project_invite_key.project.public_id == "second_public_project_id" + or project_invite_key.project.public_id == "file_testing_project" + ): + number_of_asserted_projects += 1 + assert len(project_invite_keys) == number_of_asserted_projects + assert len(project_invite_keys) == len(invited_user.unit.projects) + assert len(project_invite_keys) == 5 + + with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=token, + json=new_unit_user, + ) + # No new mail should be sent for the token and neither for an invite + assert mock_mail_send.call_count == 0 + + assert response.status_code == http.HTTPStatus.BAD_REQUEST + message = response.json.get("message") + assert "user was already added to the system" in message + + +def test_add_user_with_superadmin(client): + with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: + token = tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client) + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=token, + json=first_new_user, + ) + # One mail sent for partial token and one for the invite + assert mock_mail_send.call_count == 2 + + assert response.status_code == http.HTTPStatus.OK + + invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() + assert invited_user + assert invited_user.email == first_new_user["email"] + assert invited_user.role == first_new_user["role"] + + with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=token, + json=first_new_user, + ) + # No new mail should be sent for the token and neither for an invite + assert mock_mail_send.call_count == 0 + + assert response.status_code == http.HTTPStatus.BAD_REQUEST + message = response.json.get("message") + assert "user was already added to the system" in message + + +def test_add_user_existing_email_no_project(client): + invited_user = models.Invite.query.filter_by( + email=existing_invite["email"], role=existing_invite["role"] + ).one_or_none() + assert invited_user + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), + json=existing_invite, + ) + assert response.status_code == http.HTTPStatus.BAD_REQUEST + + +def test_add_unitadmin_user_with_unitpersonnel_permission_denied(client): + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + json=new_unit_admin, + ) + assert response.status_code == http.HTTPStatus.FORBIDDEN + + invited_user = models.Invite.query.filter_by(email=new_unit_admin["email"]).one_or_none() + assert invited_user is None + + +# Add existing users to projects ################################# Add existing users to projects # +def test_add_existing_user_without_project(client): + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + json=existing_research_user, + ) + assert response.status_code == http.HTTPStatus.BAD_REQUEST + + +def test_research_user_cannot_add_existing_user_to_existing_project(client): + user_copy = existing_research_user_to_existing_project.copy() + project_id = user_copy.pop("project") + + project = models.Project.query.filter_by(public_id=project_id).one_or_none() + user = models.Email.query.filter_by( + email=existing_research_user_to_existing_project["email"] + ).one_or_none() + project_user_before_addition = models.ProjectUsers.query.filter( + sqlalchemy.and_( + models.ProjectUsers.user_id == user.user_id, + models.ProjectUsers.project_id == project.id, + ) + ).one_or_none() + assert project_user_before_addition is None + + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["researchuser"]).token(client), + query_string={"project": project_id}, + json=user_copy, + ) + assert response.status_code == http.HTTPStatus.FORBIDDEN + + project_user_after_addition = models.ProjectUsers.query.filter( + sqlalchemy.and_( + models.ProjectUsers.user_id == user.user_id, + models.ProjectUsers.project_id == project.id, + ) + ).one_or_none() + assert project_user_after_addition is None + + +# projectowner adds researchuser2 to projects[0] +def test_project_owner_can_add_existing_user_to_existing_project(client): + user_copy = existing_research_user_to_existing_project.copy() + project_id = user_copy.pop("project") + + project = models.Project.query.filter_by(public_id=project_id).one_or_none() + user = models.Email.query.filter_by( + email=existing_research_user_to_existing_project["email"] + ).one_or_none() + project_user_before_addition = models.ProjectUsers.query.filter( + sqlalchemy.and_( + models.ProjectUsers.user_id == user.user_id, + models.ProjectUsers.project_id == project.id, + ) + ).one_or_none() + assert project_user_before_addition is None + + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["projectowner"]).token(client), + query_string={"project": project_id}, + json=user_copy, + ) + assert response.status_code == http.HTTPStatus.OK + + project_user_after_addition = models.ProjectUsers.query.filter( + sqlalchemy.and_( + models.ProjectUsers.user_id == user.user_id, + models.ProjectUsers.project_id == project.id, + ) + ).one_or_none() + assert project_user_after_addition is not None + + +def test_add_existing_user_to_existing_project(client): + user_copy = existing_research_user_to_existing_project.copy() + project_id = user_copy.pop("project") + + project = models.Project.query.filter_by(public_id=project_id).one_or_none() + user = models.Email.query.filter_by( + email=existing_research_user_to_existing_project["email"] + ).one_or_none() + project_user_before_addition = models.ProjectUsers.query.filter( + sqlalchemy.and_( + models.ProjectUsers.user_id == user.user_id, + models.ProjectUsers.project_id == project.id, + ) + ).one_or_none() + assert project_user_before_addition is None + + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + query_string={"project": project_id}, + json=user_copy, + ) + assert response.status_code == http.HTTPStatus.OK + + project_user_after_addition = models.ProjectUsers.query.filter( + sqlalchemy.and_( + models.ProjectUsers.user_id == user.user_id, + models.ProjectUsers.project_id == project.id, + ) + ).one_or_none() + assert project_user_after_addition + + +def test_add_existing_user_to_existing_project_no_mail_flag(client): + "Test that an e-mail notification is not send when the --no-mail flag is used" + + user_copy = existing_research_user_to_existing_project.copy() + project_id = user_copy.pop("project") + new_status = {"new_status": "Available"} + user_copy["send_email"] = False + token = tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client) + + # make project available prior to test, otherwise an e-mail is never sent. + response = client.post( + tests.DDSEndpoint.PROJECT_STATUS, + headers=token, + query_string={"project": project_id}, + data=json.dumps(new_status), + content_type="application/json", + ) + + # Test mail sending is suppressed + + with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: + with unittest.mock.patch.object( + dds_web.api.user.AddUser, "compose_and_send_email_to_user" + ) as mock_mail_func: + print(user_copy) + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=token, + query_string={"project": project_id}, + data=json.dumps(user_copy), + content_type="application/json", + ) + # assert that no mail is being sent. + assert mock_mail_func.called == False + assert mock_mail_send.call_count == 0 + + assert response.status_code == http.HTTPStatus.OK + assert "An e-mail notification has not been sent." in response.json["message"] + + +def test_add_existing_user_to_existing_project_after_release(client): + user_copy = existing_research_user_to_existing_project.copy() + project_id = user_copy.pop("project") + + project = models.Project.query.filter_by(public_id=project_id).one_or_none() + user = models.Email.query.filter_by( + email=existing_research_user_to_existing_project["email"] + ).one_or_none() + project_user_before_addition = models.ProjectUsers.query.filter( + sqlalchemy.and_( + models.ProjectUsers.user_id == user.user_id, + models.ProjectUsers.project_id == project.id, + ) + ).one_or_none() + assert project_user_before_addition is None + + # release project + response = client.post( + tests.DDSEndpoint.PROJECT_STATUS, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + query_string={"project": project_id}, + json={"new_status": "Available"}, + ) + assert response.status_code == http.HTTPStatus.OK + assert project.current_status == "Available" + + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + query_string={"project": project_id}, + json=user_copy, + ) + assert response.status_code == http.HTTPStatus.OK + + project_user_after_addition = models.ProjectUsers.query.filter( + sqlalchemy.and_( + models.ProjectUsers.user_id == user.user_id, + models.ProjectUsers.project_id == project.id, + ) + ).one_or_none() + assert project_user_after_addition -# def test_update_project_to_existing_invite_by_unituser(client): -# "Test that project ownership can be updated for an existing invite" +def test_add_existing_user_to_nonexistent_proj(client): + user_copy = existing_research_user_to_nonexistent_proj.copy() + project = user_copy.pop("project") + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + query_string={"project": project}, + json=user_copy, + ) + assert response.status_code == http.HTTPStatus.BAD_REQUEST -# # Create Invite upfront - -# project = existing_project -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# query_string={"project": project}, -# json=first_new_user, -# ) -# assert response.status_code == http.HTTPStatus.OK - -# project_obj = models.Project.query.filter_by(public_id=existing_project).one_or_none() -# invite_obj = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() -# project_invite = models.ProjectInviteKeys.query.filter( -# sqlalchemy.and_( -# models.ProjectInviteKeys.invite_id == invite_obj.id, -# models.ProjectUserKeys.project_id == project_obj.id, -# ) -# ).one_or_none() +def test_existing_user_change_ownership(client): + project = models.Project.query.filter_by( + public_id=change_owner_existing_user["project"] + ).one_or_none() + user = models.Email.query.filter_by(email=change_owner_existing_user["email"]).one_or_none() + project_user = models.ProjectUsers.query.filter( + sqlalchemy.and_( + models.ProjectUsers.user_id == user.user_id, + models.ProjectUsers.project_id == project.id, + ) + ).one_or_none() + + assert not project_user.owner + + user_new_owner_status = change_owner_existing_user.copy() + project = user_new_owner_status.pop("project") + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + query_string={"project": project}, + json=user_new_owner_status, + ) + + assert response.status_code == http.HTTPStatus.OK + + db.session.refresh(project_user) + + assert project_user.owner + + +def test_existing_user_change_ownership_same_permissions(client): + user_same_ownership = submit_with_same_ownership.copy() + project = user_same_ownership.pop("project") + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + query_string={"project": project}, + json=user_same_ownership, + ) + assert response.status_code == http.HTTPStatus.FORBIDDEN + + +def test_add_existing_user_with_unsuitable_role(client): + user_with_unsuitable_role = existing_research_user_to_existing_project.copy() + user_with_unsuitable_role["role"] = "Unit Admin" + project = user_with_unsuitable_role.pop("project") + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + query_string={"project": project}, + json=user_with_unsuitable_role, + ) + assert response.status_code == http.HTTPStatus.FORBIDDEN + + +# Invite to project ########################################################### Invite to project # + + +def test_invite_with_project_by_unituser(client): + "Test that a new invite including a project can be created" + + project = existing_project + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + query_string={"project": project}, + json=first_new_user, + ) + assert response.status_code == http.HTTPStatus.OK + + invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() + assert invited_user + assert invited_user.email == first_new_user["email"] + assert invited_user.role == first_new_user["role"] + + assert invited_user.nonce is not None + assert invited_user.public_key is not None + assert invited_user.private_key is not None + + project_invite_keys = invited_user.project_invite_keys + assert len(project_invite_keys) == 1 + assert project_invite_keys[0].project.public_id == project + assert not project_invite_keys[0].owner + + +def test_add_project_to_existing_invite_by_unituser(client): + "Test that a project can be associated with an existing invite" + + # Create invite upfront + + project = existing_project + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + json=first_new_user, + ) + assert response.status_code == http.HTTPStatus.OK + + invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() + + # Check that the invite has no projects yet + + assert invited_user + assert len(invited_user.project_invite_keys) == 0 + + # Add project to existing invite + + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + query_string={"project": project}, + json=first_new_user, + ) + + assert response.status_code == http.HTTPStatus.OK + + # Check that the invite has now a project association + project_invite_keys = invited_user.project_invite_keys + assert len(project_invite_keys) == 1 + assert project_invite_keys[0].project.public_id == project + assert not project_invite_keys[0].owner + + +def test_update_project_to_existing_invite_by_unituser(client): + "Test that project ownership can be updated for an existing invite" + + # Create Invite upfront + + project = existing_project + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + query_string={"project": project}, + json=first_new_user, + ) + assert response.status_code == http.HTTPStatus.OK + + project_obj = models.Project.query.filter_by(public_id=existing_project).one_or_none() + invite_obj = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() + + project_invite = models.ProjectInviteKeys.query.filter( + sqlalchemy.and_( + models.ProjectInviteKeys.invite_id == invite_obj.id, + models.ProjectUserKeys.project_id == project_obj.id, + ) + ).one_or_none() -# assert project_invite -# assert not project_invite.owner + assert project_invite + assert not project_invite.owner -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# query_string={"project": project}, -# json=first_new_owner, -# ) + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + query_string={"project": project}, + json=first_new_owner, + ) -# assert response.status_code == http.HTTPStatus.OK - -# db.session.refresh(project_invite) + assert response.status_code == http.HTTPStatus.OK -# assert project_invite.owner - - -# def test_invited_as_owner_and_researcher_to_different_project(client): -# "Test that an invite can be owner of one project and researcher of another" + db.session.refresh(project_invite) -# # Create Invite upfront as owner + assert project_invite.owner -# project = existing_project -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# query_string={"project": project}, -# json=first_new_owner, -# ) -# assert response.status_code == http.HTTPStatus.OK -# # Perform second invite as researcher -# project2 = existing_project_2 -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# query_string={"project": project2}, -# json=first_new_user, -# ) -# assert response.status_code == http.HTTPStatus.OK +def test_invited_as_owner_and_researcher_to_different_project(client): + "Test that an invite can be owner of one project and researcher of another" -# project_obj_owner = models.Project.query.filter_by(public_id=existing_project).one_or_none() -# project_obj_not_owner = models.Project.query.filter_by( -# public_id=existing_project_2 -# ).one_or_none() + # Create Invite upfront as owner + + project = existing_project + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + query_string={"project": project}, + json=first_new_owner, + ) + assert response.status_code == http.HTTPStatus.OK + + # Perform second invite as researcher + project2 = existing_project_2 + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + query_string={"project": project2}, + json=first_new_user, + ) + assert response.status_code == http.HTTPStatus.OK + + project_obj_owner = models.Project.query.filter_by(public_id=existing_project).one_or_none() + project_obj_not_owner = models.Project.query.filter_by( + public_id=existing_project_2 + ).one_or_none() -# invite_obj = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() + invite_obj = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() -# project_invite_owner = models.ProjectInviteKeys.query.filter( -# sqlalchemy.and_( -# models.ProjectInviteKeys.invite_id == invite_obj.id, -# models.ProjectInviteKeys.project_id == project_obj_owner.id, -# ) -# ).one_or_none() + project_invite_owner = models.ProjectInviteKeys.query.filter( + sqlalchemy.and_( + models.ProjectInviteKeys.invite_id == invite_obj.id, + models.ProjectInviteKeys.project_id == project_obj_owner.id, + ) + ).one_or_none() -# assert project_invite_owner -# assert project_invite_owner.owner + assert project_invite_owner + assert project_invite_owner.owner -# project_invite_not_owner = models.ProjectInviteKeys.query.filter( -# sqlalchemy.and_( -# models.ProjectInviteKeys.invite_id == invite_obj.id, -# models.ProjectInviteKeys.project_id == project_obj_not_owner.id, -# ) -# ).one_or_none() - -# assert project_invite_not_owner -# assert not project_invite_not_owner.owner - -# # Owner or not should not be stored on the invite -# assert invite_obj.role == "Researcher" - - -# def test_invite_to_project_by_project_owner(client): -# "Test that a project owner can invite to its project" + project_invite_not_owner = models.ProjectInviteKeys.query.filter( + sqlalchemy.and_( + models.ProjectInviteKeys.invite_id == invite_obj.id, + models.ProjectInviteKeys.project_id == project_obj_not_owner.id, + ) + ).one_or_none() -# project = existing_project -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["projectowner"]).token(client), -# query_string={"project": project}, -# json=first_new_user, -# ) -# assert response.status_code == http.HTTPStatus.OK - -# invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() -# assert invited_user -# assert invited_user.email == first_new_user["email"] -# assert invited_user.role == first_new_user["role"] - -# assert invited_user.nonce is not None -# assert invited_user.public_key is not None -# assert invited_user.private_key is not None - -# project_invite_keys = invited_user.project_invite_keys -# assert len(project_invite_keys) == 1 -# assert project_invite_keys[0].project.public_id == project -# assert not project_invite_keys[0].owner + assert project_invite_not_owner + assert not project_invite_not_owner.owner + # Owner or not should not be stored on the invite + assert invite_obj.role == "Researcher" -def test_add_researcher_and_owner_to_project_with_superadmin(client): + +def test_invite_to_project_by_project_owner(client): + "Test that a project owner can invite to its project" + + project = existing_project + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["projectowner"]).token(client), + query_string={"project": project}, + json=first_new_user, + ) + assert response.status_code == http.HTTPStatus.OK + + invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() + assert invited_user + assert invited_user.email == first_new_user["email"] + assert invited_user.role == first_new_user["role"] + + assert invited_user.nonce is not None + assert invited_user.public_key is not None + assert invited_user.private_key is not None + + project_invite_keys = invited_user.project_invite_keys + assert len(project_invite_keys) == 1 + assert project_invite_keys[0].project.public_id == project + assert not project_invite_keys[0].owner + + +def test_add_anyuser_to_project_with_superadmin(client): + """Super admins cannot invite to project.""" project = existing_project for x in [first_new_user, first_new_owner, new_unit_user, new_unit_admin]: response = client.post( @@ -717,11 +718,13 @@ def test_add_researcher_and_owner_to_project_with_superadmin(client): ) assert response.status_code == http.HTTPStatus.BAD_REQUEST + # An invite should not have been created invited_user = models.Invite.query.filter_by(email=x["email"]).one_or_none() assert not invited_user def test_add_unituser_and_admin_no_unit_with_superadmin(client): + """A super admin needs to specify a unit to be able to invite unit users.""" project = existing_project for x in [new_unit_user, new_unit_admin]: response = client.post( @@ -732,85 +735,85 @@ def test_add_unituser_and_admin_no_unit_with_superadmin(client): assert "You need to specify a unit" in response.json["message"] assert response.status_code == http.HTTPStatus.BAD_REQUEST + invited_user = models.Invite.query.filter_by(email=x["email"]).one_or_none() assert not invited_user -def test_add_researchuser_project_no_access_unit_admin(client): +def test_add_researchuser_project_no_access_unit_admin_and_personnel(client): + """A unit admin and personnel should not be able to give access to a project + which they themselves have lost access to.""" + # Make sure the project exists project = models.Project.query.filter_by(public_id=existing_project).one_or_none() assert project - project_user_key = models.ProjectUserKeys.query.filter_by( - user_id="unitadmin", project_id=project.id - ).one_or_none() - assert project_user_key - - db.session.delete(project_user_key) - project_user_key = models.ProjectUserKeys.query.filter_by( - user_id="unitadmin", project_id=project.id - ).one_or_none() - assert not project_user_key - - for x in [first_new_user, first_new_owner]: - - invited_user = models.Invite.query.filter_by(email=x["email"]).one_or_none() - print(invited_user) - assert not invited_user - + for inviter in ["unitadmin", "unituser", "projectowner"]: + # Check that the unit admin has access to the project first + project_user_key = models.ProjectUserKeys.query.filter_by( + user_id=inviter, project_id=project.id + ).one_or_none() + assert project_user_key + + # Remove the project access (for test) + db.session.delete(project_user_key) + + # Make sure the project access does not exist now + project_user_key = models.ProjectUserKeys.query.filter_by( + user_id=inviter, project_id=project.id + ).one_or_none() + assert not project_user_key + + for x in [first_new_user, first_new_owner]: + # Make sure there is no ongoing invite + invited_user_before = models.Invite.query.filter_by(email=x["email"]).one_or_none() + if invited_user_before: + db.session.delete(invited_user_before) + invited_user_before = models.Invite.query.filter_by(email=x["email"]).one_or_none() + assert not invited_user_before + + # Attempt invite + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS[inviter]).token(client), + json=x, + query_string={"project": project.public_id}, + ) + + # The invite should still be done, but they can't invite to a project + invited_user = models.Invite.query.filter_by(email=x["email"]).one_or_none() + assert invited_user + + # Make sure there are no project v + assert not invited_user.project_invite_keys + + # There should be an error message + assert response.status_code == http.HTTPStatus.BAD_REQUEST + assert "The user could not be added to the project(s)" in response.json["message"] + + # Verify ok error messages + assert "errors" in response.json + assert project.public_id in response.json["errors"] + assert ( + "You do not have access to the specified project." + in response.json["errors"][project.public_id] + ) + + +# Invite without email +def test_invite_without_email(client): + """The email is required.""" + user_no_email = first_new_user.copy() + user_no_email.pop("email") + + for inviter in ["superadmin", "unitadmin", "unituser"]: + # Attempt invite response = client.post( tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), - json=x, - query_string={"project": project.public_id}, + headers=tests.UserAuth(tests.USER_CREDENTIALS[inviter]).token(client), + json=user_no_email, + # query_string={"project": existing_project}, ) + # There should be an error message assert response.status_code == http.HTTPStatus.BAD_REQUEST - assert "The user could not be added to the project(s)" in response.json["message"] - - assert "errors" in response.json - assert project.public_id in response.json["errors"] - assert ( - "You do not have access to the specified project." - in response.json["errors"][project.public_id] - ) - - invited_user = models.Invite.query.filter_by(email=x["email"]).one_or_none() - assert invited_user - - -# def test_add_unituser_project_no_access_unit_admin(client): -# project = models.Project.query.filter_by(public_id=existing_project).one_or_none() -# assert project - -# project_user_key = models.ProjectUserKeys.query.filter_by( -# user_id="unitadmin", project_id=project.id -# ).one_or_none() -# assert project_user_key - -# db.session.delete(project_user_key) -# project_user_key = models.ProjectUserKeys.query.filter_by( -# user_id="unitadmin", project_id=project.id -# ).one_or_none() -# assert not project_user_key - -# for x in [first_new_user, first_new_owner]: - -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), -# json=x, -# query_string={"project": project.public_id}, -# ) - -# assert response.status_code == http.HTTPStatus.BAD_REQUEST -# assert "The user could not be added to the project(s)" in response.json["message"] - -# assert "errors" in response.json -# assert project.public_id in response.json["errors"] -# assert ( -# "You do not have access to the specified project." -# in response.json["errors"][project.public_id] -# ) - -# invited_user = models.Invite.query.filter_by(email=x["email"]).one_or_none() -# assert not invited_user + assert "Email address required to add or invite." in response.json["message"] From c0d4374a9d77f6bb526e779478d46ce4e95b7463 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Sun, 13 Mar 2022 11:28:42 +0100 Subject: [PATCH 046/293] test for invite superadmin as unitadmin --- tests/test_user_add.py | 1539 ++++++++++++++++++++-------------------- 1 file changed, 777 insertions(+), 762 deletions(-) diff --git a/tests/test_user_add.py b/tests/test_user_add.py index 46f4fd580..6b0f640fc 100644 --- a/tests/test_user_add.py +++ b/tests/test_user_add.py @@ -22,6 +22,7 @@ first_new_user_invalid_email = {"email": "first_invalid_email", "role": first_new_user["role"]} existing_invite = {"email": "existing_invite_email@mailtrap.io", "role": "Researcher"} new_unit_admin = {"email": "new_unit_admin@mailtrap.io", "role": "Unit Admin"} +new_super_admin = {"email": "new_super_admin@mailtrap.io", "role": "Super Admin"} new_unit_user = {"email": "new_unit_user@mailtrap.io", "role": "Unit Personnel"} existing_research_user = {"email": "researchuser2@mailtrap.io", "role": "Researcher"} existing_research_user_owner = {"email": "researchuser2@mailtrap.io", "role": "Project Owner"} @@ -43,777 +44,791 @@ "project": "second_public_project_id", } -# Inviting Users ################################################################# Inviting Users # -def test_add_user_with_researcher(client): - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["researchuser"]).token(client), - json=first_new_user, - ) - assert response.status_code == http.HTTPStatus.FORBIDDEN - invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() - assert invited_user is None +# # Inviting Users ################################################################# Inviting Users # +# def test_add_user_with_researcher(client): +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["researchuser"]).token(client), +# json=first_new_user, +# ) +# assert response.status_code == http.HTTPStatus.FORBIDDEN +# invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() +# assert invited_user is None + + +# def test_add_user_with_unituser_no_role(client): +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), +# json=first_new_email, +# ) +# assert response.status_code == http.HTTPStatus.BAD_REQUEST +# invited_user = models.Invite.query.filter_by(email=first_new_email["email"]).one_or_none() +# assert invited_user is None + + +# def test_add_user_with_unitadmin_with_extraargs(client): +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), +# json=first_new_user_extra_args, +# ) +# assert response.status_code == http.HTTPStatus.OK +# invited_user = models.Invite.query.filter_by( +# email=first_new_user_extra_args["email"] +# ).one_or_none() +# assert invited_user + + +# def test_add_user_with_unitadmin_and_invalid_role(client): +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), +# json=first_new_user_invalid_role, +# ) +# assert response.status_code == http.HTTPStatus.BAD_REQUEST +# invited_user = models.Invite.query.filter_by( +# email=first_new_user_invalid_role["email"] +# ).one_or_none() +# assert invited_user is None + + +# def test_add_user_with_unitadmin_and_invalid_email(client): +# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), +# json=first_new_user_invalid_email, +# ) +# assert response.status_code == http.HTTPStatus.BAD_REQUEST +# # An email is always sent when receiving the partial token +# mock_mail_send.assert_called_once() + +# invited_user = models.Invite.query.filter_by( +# email=first_new_user_invalid_email["email"] +# ).one_or_none() +# assert invited_user is None + + +# def test_add_user_with_unitadmin(client): +# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: +# token = tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client) +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=token, +# json=first_new_user, +# ) +# # One mail sent for partial token and one for the invite +# assert mock_mail_send.call_count == 2 + +# assert response.status_code == http.HTTPStatus.OK + +# invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() +# assert invited_user +# assert invited_user.email == first_new_user["email"] +# assert invited_user.role == first_new_user["role"] + +# assert invited_user.nonce is not None +# assert invited_user.public_key is not None +# assert invited_user.private_key is not None +# assert invited_user.project_invite_keys == [] + +# # Repeating the invite should not send a new invite: +# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=token, +# json=first_new_user, +# ) +# # No new mail should be sent for the token and neither for an invite +# assert mock_mail_send.call_count == 0 +# assert response.status_code == http.HTTPStatus.BAD_REQUEST +# message = response.json.get("message") +# assert "user was already added to the system" in message + + +# def test_add_unit_user_with_unitadmin(client): + +# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: +# token = tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client) +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=token, +# json=new_unit_user, +# ) +# # One mail sent for partial token and one for the invite +# assert mock_mail_send.call_count == 2 + +# assert response.status_code == http.HTTPStatus.OK + +# invited_user = models.Invite.query.filter_by(email=new_unit_user["email"]).one_or_none() +# assert invited_user +# assert invited_user.email == new_unit_user["email"] +# assert invited_user.role == new_unit_user["role"] + +# assert invited_user.nonce is not None +# assert invited_user.public_key is not None +# assert invited_user.private_key is not None + +# project_invite_keys = invited_user.project_invite_keys +# number_of_asserted_projects = 0 +# for project_invite_key in project_invite_keys: +# if ( +# project_invite_key.project.public_id == "public_project_id" +# or project_invite_key.project.public_id == "unused_project_id" +# or project_invite_key.project.public_id == "restricted_project_id" +# or project_invite_key.project.public_id == "second_public_project_id" +# or project_invite_key.project.public_id == "file_testing_project" +# ): +# number_of_asserted_projects += 1 +# assert len(project_invite_keys) == number_of_asserted_projects +# assert len(project_invite_keys) == len(invited_user.unit.projects) +# assert len(project_invite_keys) == 5 + +# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=token, +# json=new_unit_user, +# ) +# # No new mail should be sent for the token and neither for an invite +# assert mock_mail_send.call_count == 0 + +# assert response.status_code == http.HTTPStatus.BAD_REQUEST +# message = response.json.get("message") +# assert "user was already added to the system" in message + + +# def test_add_user_with_superadmin(client): +# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: +# token = tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client) +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=token, +# json=first_new_user, +# ) +# # One mail sent for partial token and one for the invite +# assert mock_mail_send.call_count == 2 + +# assert response.status_code == http.HTTPStatus.OK + +# invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() +# assert invited_user +# assert invited_user.email == first_new_user["email"] +# assert invited_user.role == first_new_user["role"] + +# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=token, +# json=first_new_user, +# ) +# # No new mail should be sent for the token and neither for an invite +# assert mock_mail_send.call_count == 0 + +# assert response.status_code == http.HTTPStatus.BAD_REQUEST +# message = response.json.get("message") +# assert "user was already added to the system" in message + + +# def test_add_user_existing_email_no_project(client): +# invited_user = models.Invite.query.filter_by( +# email=existing_invite["email"], role=existing_invite["role"] +# ).one_or_none() +# assert invited_user +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), +# json=existing_invite, +# ) +# assert response.status_code == http.HTTPStatus.BAD_REQUEST + + +# def test_add_unitadmin_user_with_unitpersonnel_permission_denied(client): +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# json=new_unit_admin, +# ) +# assert response.status_code == http.HTTPStatus.FORBIDDEN + +# invited_user = models.Invite.query.filter_by(email=new_unit_admin["email"]).one_or_none() +# assert invited_user is None + + +# # Add existing users to projects ################################# Add existing users to projects # +# def test_add_existing_user_without_project(client): +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# json=existing_research_user, +# ) +# assert response.status_code == http.HTTPStatus.BAD_REQUEST + + +# def test_research_user_cannot_add_existing_user_to_existing_project(client): +# user_copy = existing_research_user_to_existing_project.copy() +# project_id = user_copy.pop("project") + +# project = models.Project.query.filter_by(public_id=project_id).one_or_none() +# user = models.Email.query.filter_by( +# email=existing_research_user_to_existing_project["email"] +# ).one_or_none() +# project_user_before_addition = models.ProjectUsers.query.filter( +# sqlalchemy.and_( +# models.ProjectUsers.user_id == user.user_id, +# models.ProjectUsers.project_id == project.id, +# ) +# ).one_or_none() +# assert project_user_before_addition is None + +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["researchuser"]).token(client), +# query_string={"project": project_id}, +# json=user_copy, +# ) +# assert response.status_code == http.HTTPStatus.FORBIDDEN + +# project_user_after_addition = models.ProjectUsers.query.filter( +# sqlalchemy.and_( +# models.ProjectUsers.user_id == user.user_id, +# models.ProjectUsers.project_id == project.id, +# ) +# ).one_or_none() +# assert project_user_after_addition is None + + +# # projectowner adds researchuser2 to projects[0] +# def test_project_owner_can_add_existing_user_to_existing_project(client): +# user_copy = existing_research_user_to_existing_project.copy() +# project_id = user_copy.pop("project") + +# project = models.Project.query.filter_by(public_id=project_id).one_or_none() +# user = models.Email.query.filter_by( +# email=existing_research_user_to_existing_project["email"] +# ).one_or_none() +# project_user_before_addition = models.ProjectUsers.query.filter( +# sqlalchemy.and_( +# models.ProjectUsers.user_id == user.user_id, +# models.ProjectUsers.project_id == project.id, +# ) +# ).one_or_none() +# assert project_user_before_addition is None + +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["projectowner"]).token(client), +# query_string={"project": project_id}, +# json=user_copy, +# ) +# assert response.status_code == http.HTTPStatus.OK + +# project_user_after_addition = models.ProjectUsers.query.filter( +# sqlalchemy.and_( +# models.ProjectUsers.user_id == user.user_id, +# models.ProjectUsers.project_id == project.id, +# ) +# ).one_or_none() +# assert project_user_after_addition is not None + + +# def test_add_existing_user_to_existing_project(client): +# user_copy = existing_research_user_to_existing_project.copy() +# project_id = user_copy.pop("project") + +# project = models.Project.query.filter_by(public_id=project_id).one_or_none() +# user = models.Email.query.filter_by( +# email=existing_research_user_to_existing_project["email"] +# ).one_or_none() +# project_user_before_addition = models.ProjectUsers.query.filter( +# sqlalchemy.and_( +# models.ProjectUsers.user_id == user.user_id, +# models.ProjectUsers.project_id == project.id, +# ) +# ).one_or_none() +# assert project_user_before_addition is None + +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# query_string={"project": project_id}, +# json=user_copy, +# ) +# assert response.status_code == http.HTTPStatus.OK + +# project_user_after_addition = models.ProjectUsers.query.filter( +# sqlalchemy.and_( +# models.ProjectUsers.user_id == user.user_id, +# models.ProjectUsers.project_id == project.id, +# ) +# ).one_or_none() +# assert project_user_after_addition + + +# def test_add_existing_user_to_existing_project_no_mail_flag(client): +# "Test that an e-mail notification is not send when the --no-mail flag is used" + +# user_copy = existing_research_user_to_existing_project.copy() +# project_id = user_copy.pop("project") +# new_status = {"new_status": "Available"} +# user_copy["send_email"] = False +# token = tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client) + +# # make project available prior to test, otherwise an e-mail is never sent. +# response = client.post( +# tests.DDSEndpoint.PROJECT_STATUS, +# headers=token, +# query_string={"project": project_id}, +# data=json.dumps(new_status), +# content_type="application/json", +# ) + +# # Test mail sending is suppressed + +# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: +# with unittest.mock.patch.object( +# dds_web.api.user.AddUser, "compose_and_send_email_to_user" +# ) as mock_mail_func: +# print(user_copy) +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=token, +# query_string={"project": project_id}, +# data=json.dumps(user_copy), +# content_type="application/json", +# ) +# # assert that no mail is being sent. +# assert mock_mail_func.called == False +# assert mock_mail_send.call_count == 0 + +# assert response.status_code == http.HTTPStatus.OK +# assert "An e-mail notification has not been sent." in response.json["message"] + + +# def test_add_existing_user_to_existing_project_after_release(client): +# user_copy = existing_research_user_to_existing_project.copy() +# project_id = user_copy.pop("project") + +# project = models.Project.query.filter_by(public_id=project_id).one_or_none() +# user = models.Email.query.filter_by( +# email=existing_research_user_to_existing_project["email"] +# ).one_or_none() +# project_user_before_addition = models.ProjectUsers.query.filter( +# sqlalchemy.and_( +# models.ProjectUsers.user_id == user.user_id, +# models.ProjectUsers.project_id == project.id, +# ) +# ).one_or_none() +# assert project_user_before_addition is None + +# # release project +# response = client.post( +# tests.DDSEndpoint.PROJECT_STATUS, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# query_string={"project": project_id}, +# json={"new_status": "Available"}, +# ) +# assert response.status_code == http.HTTPStatus.OK +# assert project.current_status == "Available" + +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# query_string={"project": project_id}, +# json=user_copy, +# ) +# assert response.status_code == http.HTTPStatus.OK + +# project_user_after_addition = models.ProjectUsers.query.filter( +# sqlalchemy.and_( +# models.ProjectUsers.user_id == user.user_id, +# models.ProjectUsers.project_id == project.id, +# ) +# ).one_or_none() +# assert project_user_after_addition + + +# def test_add_existing_user_to_nonexistent_proj(client): +# user_copy = existing_research_user_to_nonexistent_proj.copy() +# project = user_copy.pop("project") +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# query_string={"project": project}, +# json=user_copy, +# ) +# assert response.status_code == http.HTTPStatus.BAD_REQUEST + + +# def test_existing_user_change_ownership(client): +# project = models.Project.query.filter_by( +# public_id=change_owner_existing_user["project"] +# ).one_or_none() +# user = models.Email.query.filter_by(email=change_owner_existing_user["email"]).one_or_none() +# project_user = models.ProjectUsers.query.filter( +# sqlalchemy.and_( +# models.ProjectUsers.user_id == user.user_id, +# models.ProjectUsers.project_id == project.id, +# ) +# ).one_or_none() + +# assert not project_user.owner + +# user_new_owner_status = change_owner_existing_user.copy() +# project = user_new_owner_status.pop("project") +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# query_string={"project": project}, +# json=user_new_owner_status, +# ) + +# assert response.status_code == http.HTTPStatus.OK + +# db.session.refresh(project_user) + +# assert project_user.owner + + +# def test_existing_user_change_ownership_same_permissions(client): +# user_same_ownership = submit_with_same_ownership.copy() +# project = user_same_ownership.pop("project") +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# query_string={"project": project}, +# json=user_same_ownership, +# ) +# assert response.status_code == http.HTTPStatus.FORBIDDEN + + +# def test_add_existing_user_with_unsuitable_role(client): +# user_with_unsuitable_role = existing_research_user_to_existing_project.copy() +# user_with_unsuitable_role["role"] = "Unit Admin" +# project = user_with_unsuitable_role.pop("project") +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# query_string={"project": project}, +# json=user_with_unsuitable_role, +# ) +# assert response.status_code == http.HTTPStatus.FORBIDDEN + + +# # Invite to project ########################################################### Invite to project # + + +# def test_invite_with_project_by_unituser(client): +# "Test that a new invite including a project can be created" + +# project = existing_project +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# query_string={"project": project}, +# json=first_new_user, +# ) +# assert response.status_code == http.HTTPStatus.OK + +# invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() +# assert invited_user +# assert invited_user.email == first_new_user["email"] +# assert invited_user.role == first_new_user["role"] + +# assert invited_user.nonce is not None +# assert invited_user.public_key is not None +# assert invited_user.private_key is not None + +# project_invite_keys = invited_user.project_invite_keys +# assert len(project_invite_keys) == 1 +# assert project_invite_keys[0].project.public_id == project +# assert not project_invite_keys[0].owner + + +# def test_add_project_to_existing_invite_by_unituser(client): +# "Test that a project can be associated with an existing invite" + +# # Create invite upfront + +# project = existing_project +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# json=first_new_user, +# ) +# assert response.status_code == http.HTTPStatus.OK + +# invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() + +# # Check that the invite has no projects yet + +# assert invited_user +# assert len(invited_user.project_invite_keys) == 0 + +# # Add project to existing invite + +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# query_string={"project": project}, +# json=first_new_user, +# ) + +# assert response.status_code == http.HTTPStatus.OK + +# # Check that the invite has now a project association +# project_invite_keys = invited_user.project_invite_keys +# assert len(project_invite_keys) == 1 +# assert project_invite_keys[0].project.public_id == project +# assert not project_invite_keys[0].owner -def test_add_user_with_unituser_no_role(client): - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), - json=first_new_email, - ) - assert response.status_code == http.HTTPStatus.BAD_REQUEST - invited_user = models.Invite.query.filter_by(email=first_new_email["email"]).one_or_none() - assert invited_user is None +# def test_update_project_to_existing_invite_by_unituser(client): +# "Test that project ownership can be updated for an existing invite" +# # Create Invite upfront + +# project = existing_project +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# query_string={"project": project}, +# json=first_new_user, +# ) +# assert response.status_code == http.HTTPStatus.OK + +# project_obj = models.Project.query.filter_by(public_id=existing_project).one_or_none() +# invite_obj = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() -def test_add_user_with_unitadmin_with_extraargs(client): - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), - json=first_new_user_extra_args, - ) - assert response.status_code == http.HTTPStatus.OK - invited_user = models.Invite.query.filter_by( - email=first_new_user_extra_args["email"] - ).one_or_none() - assert invited_user +# project_invite = models.ProjectInviteKeys.query.filter( +# sqlalchemy.and_( +# models.ProjectInviteKeys.invite_id == invite_obj.id, +# models.ProjectUserKeys.project_id == project_obj.id, +# ) +# ).one_or_none() +# assert project_invite +# assert not project_invite.owner -def test_add_user_with_unitadmin_and_invalid_role(client): - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), - json=first_new_user_invalid_role, - ) - assert response.status_code == http.HTTPStatus.BAD_REQUEST - invited_user = models.Invite.query.filter_by( - email=first_new_user_invalid_role["email"] - ).one_or_none() - assert invited_user is None - - -def test_add_user_with_unitadmin_and_invalid_email(client): - with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), - json=first_new_user_invalid_email, - ) - assert response.status_code == http.HTTPStatus.BAD_REQUEST - # An email is always sent when receiving the partial token - mock_mail_send.assert_called_once() - - invited_user = models.Invite.query.filter_by( - email=first_new_user_invalid_email["email"] - ).one_or_none() - assert invited_user is None - - -def test_add_user_with_unitadmin(client): - with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: - token = tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client) - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=token, - json=first_new_user, - ) - # One mail sent for partial token and one for the invite - assert mock_mail_send.call_count == 2 - - assert response.status_code == http.HTTPStatus.OK - - invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() - assert invited_user - assert invited_user.email == first_new_user["email"] - assert invited_user.role == first_new_user["role"] - - assert invited_user.nonce is not None - assert invited_user.public_key is not None - assert invited_user.private_key is not None - assert invited_user.project_invite_keys == [] - - # Repeating the invite should not send a new invite: - with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=token, - json=first_new_user, - ) - # No new mail should be sent for the token and neither for an invite - assert mock_mail_send.call_count == 0 - assert response.status_code == http.HTTPStatus.BAD_REQUEST - message = response.json.get("message") - assert "user was already added to the system" in message - - -def test_add_unit_user_with_unitadmin(client): - - with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: - token = tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client) - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=token, - json=new_unit_user, - ) - # One mail sent for partial token and one for the invite - assert mock_mail_send.call_count == 2 - - assert response.status_code == http.HTTPStatus.OK - - invited_user = models.Invite.query.filter_by(email=new_unit_user["email"]).one_or_none() - assert invited_user - assert invited_user.email == new_unit_user["email"] - assert invited_user.role == new_unit_user["role"] - - assert invited_user.nonce is not None - assert invited_user.public_key is not None - assert invited_user.private_key is not None - - project_invite_keys = invited_user.project_invite_keys - number_of_asserted_projects = 0 - for project_invite_key in project_invite_keys: - if ( - project_invite_key.project.public_id == "public_project_id" - or project_invite_key.project.public_id == "unused_project_id" - or project_invite_key.project.public_id == "restricted_project_id" - or project_invite_key.project.public_id == "second_public_project_id" - or project_invite_key.project.public_id == "file_testing_project" - ): - number_of_asserted_projects += 1 - assert len(project_invite_keys) == number_of_asserted_projects - assert len(project_invite_keys) == len(invited_user.unit.projects) - assert len(project_invite_keys) == 5 - - with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=token, - json=new_unit_user, - ) - # No new mail should be sent for the token and neither for an invite - assert mock_mail_send.call_count == 0 - - assert response.status_code == http.HTTPStatus.BAD_REQUEST - message = response.json.get("message") - assert "user was already added to the system" in message - - -def test_add_user_with_superadmin(client): - with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: - token = tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client) - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=token, - json=first_new_user, - ) - # One mail sent for partial token and one for the invite - assert mock_mail_send.call_count == 2 - - assert response.status_code == http.HTTPStatus.OK - - invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() - assert invited_user - assert invited_user.email == first_new_user["email"] - assert invited_user.role == first_new_user["role"] - - with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=token, - json=first_new_user, - ) - # No new mail should be sent for the token and neither for an invite - assert mock_mail_send.call_count == 0 - - assert response.status_code == http.HTTPStatus.BAD_REQUEST - message = response.json.get("message") - assert "user was already added to the system" in message - - -def test_add_user_existing_email_no_project(client): - invited_user = models.Invite.query.filter_by( - email=existing_invite["email"], role=existing_invite["role"] - ).one_or_none() - assert invited_user +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# query_string={"project": project}, +# json=first_new_owner, +# ) + +# assert response.status_code == http.HTTPStatus.OK + +# db.session.refresh(project_invite) + +# assert project_invite.owner + + +# def test_invited_as_owner_and_researcher_to_different_project(client): +# "Test that an invite can be owner of one project and researcher of another" + +# # Create Invite upfront as owner + +# project = existing_project +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# query_string={"project": project}, +# json=first_new_owner, +# ) +# assert response.status_code == http.HTTPStatus.OK + +# # Perform second invite as researcher +# project2 = existing_project_2 +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), +# query_string={"project": project2}, +# json=first_new_user, +# ) +# assert response.status_code == http.HTTPStatus.OK + +# project_obj_owner = models.Project.query.filter_by(public_id=existing_project).one_or_none() +# project_obj_not_owner = models.Project.query.filter_by( +# public_id=existing_project_2 +# ).one_or_none() + +# invite_obj = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() + +# project_invite_owner = models.ProjectInviteKeys.query.filter( +# sqlalchemy.and_( +# models.ProjectInviteKeys.invite_id == invite_obj.id, +# models.ProjectInviteKeys.project_id == project_obj_owner.id, +# ) +# ).one_or_none() + +# assert project_invite_owner +# assert project_invite_owner.owner + +# project_invite_not_owner = models.ProjectInviteKeys.query.filter( +# sqlalchemy.and_( +# models.ProjectInviteKeys.invite_id == invite_obj.id, +# models.ProjectInviteKeys.project_id == project_obj_not_owner.id, +# ) +# ).one_or_none() + +# assert project_invite_not_owner +# assert not project_invite_not_owner.owner + +# # Owner or not should not be stored on the invite +# assert invite_obj.role == "Researcher" + + +# def test_invite_to_project_by_project_owner(client): +# "Test that a project owner can invite to its project" + +# project = existing_project +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["projectowner"]).token(client), +# query_string={"project": project}, +# json=first_new_user, +# ) +# assert response.status_code == http.HTTPStatus.OK + +# invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() +# assert invited_user +# assert invited_user.email == first_new_user["email"] +# assert invited_user.role == first_new_user["role"] + +# assert invited_user.nonce is not None +# assert invited_user.public_key is not None +# assert invited_user.private_key is not None + +# project_invite_keys = invited_user.project_invite_keys +# assert len(project_invite_keys) == 1 +# assert project_invite_keys[0].project.public_id == project +# assert not project_invite_keys[0].owner + + +# def test_add_anyuser_to_project_with_superadmin(client): +# """Super admins cannot invite to project.""" +# project = existing_project +# for x in [first_new_user, first_new_owner, new_unit_user, new_unit_admin]: +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client), +# query_string={"project": project}, +# json=x, +# ) +# assert response.status_code == http.HTTPStatus.BAD_REQUEST + +# # An invite should not have been created +# invited_user = models.Invite.query.filter_by(email=x["email"]).one_or_none() +# assert not invited_user + + +# def test_add_unituser_and_admin_no_unit_with_superadmin(client): +# """A super admin needs to specify a unit to be able to invite unit users.""" +# project = existing_project +# for x in [new_unit_user, new_unit_admin]: +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client), +# json=x, +# ) + +# assert "You need to specify a unit" in response.json["message"] +# assert response.status_code == http.HTTPStatus.BAD_REQUEST + +# invited_user = models.Invite.query.filter_by(email=x["email"]).one_or_none() +# assert not invited_user + + +# def test_add_researchuser_project_no_access_unit_admin_and_personnel(client): +# """A unit admin and personnel should not be able to give access to a project +# which they themselves have lost access to.""" +# # Make sure the project exists +# project = models.Project.query.filter_by(public_id=existing_project).one_or_none() +# assert project + +# for inviter in ["unitadmin", "unituser", "projectowner"]: +# # Check that the unit admin has access to the project first +# project_user_key = models.ProjectUserKeys.query.filter_by( +# user_id=inviter, project_id=project.id +# ).one_or_none() +# assert project_user_key + +# # Remove the project access (for test) +# db.session.delete(project_user_key) + +# # Make sure the project access does not exist now +# project_user_key = models.ProjectUserKeys.query.filter_by( +# user_id=inviter, project_id=project.id +# ).one_or_none() +# assert not project_user_key + +# for x in [first_new_user, first_new_owner]: +# # Make sure there is no ongoing invite +# invited_user_before = models.Invite.query.filter_by(email=x["email"]).one_or_none() +# if invited_user_before: +# db.session.delete(invited_user_before) +# invited_user_before = models.Invite.query.filter_by(email=x["email"]).one_or_none() +# assert not invited_user_before + +# # Attempt invite +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS[inviter]).token(client), +# json=x, +# query_string={"project": project.public_id}, +# ) + +# # The invite should still be done, but they can't invite to a project +# invited_user = models.Invite.query.filter_by(email=x["email"]).one_or_none() +# assert invited_user + +# # Make sure there are no project v +# assert not invited_user.project_invite_keys + +# # There should be an error message +# assert response.status_code == http.HTTPStatus.BAD_REQUEST +# assert "The user could not be added to the project(s)" in response.json["message"] + +# # Verify ok error messages +# assert "errors" in response.json +# assert project.public_id in response.json["errors"] +# assert ( +# "You do not have access to the specified project." +# in response.json["errors"][project.public_id] +# ) + + +# # Invite without email +# def test_invite_without_email(client): +# """The email is required.""" +# user_no_email = first_new_user.copy() +# user_no_email.pop("email") + +# for inviter in ["superadmin", "unitadmin", "unituser"]: +# # Attempt invite +# response = client.post( +# tests.DDSEndpoint.USER_ADD, +# headers=tests.UserAuth(tests.USER_CREDENTIALS[inviter]).token(client), +# json=user_no_email, +# # query_string={"project": existing_project}, +# ) + +# # There should be an error message +# assert response.status_code == http.HTTPStatus.BAD_REQUEST +# assert "Email address required to add or invite." in response.json["message"] + + +# Invite super admin with unit admin +def test_invite_superadmin_as_unitadmin(client): + """A unit admin cannt invie a superadmin""" + # Attempt invite response = client.post( tests.DDSEndpoint.USER_ADD, headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), - json=existing_invite, - ) - assert response.status_code == http.HTTPStatus.BAD_REQUEST - - -def test_add_unitadmin_user_with_unitpersonnel_permission_denied(client): - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - json=new_unit_admin, - ) - assert response.status_code == http.HTTPStatus.FORBIDDEN - - invited_user = models.Invite.query.filter_by(email=new_unit_admin["email"]).one_or_none() - assert invited_user is None - - -# Add existing users to projects ################################# Add existing users to projects # -def test_add_existing_user_without_project(client): - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - json=existing_research_user, - ) - assert response.status_code == http.HTTPStatus.BAD_REQUEST - - -def test_research_user_cannot_add_existing_user_to_existing_project(client): - user_copy = existing_research_user_to_existing_project.copy() - project_id = user_copy.pop("project") - - project = models.Project.query.filter_by(public_id=project_id).one_or_none() - user = models.Email.query.filter_by( - email=existing_research_user_to_existing_project["email"] - ).one_or_none() - project_user_before_addition = models.ProjectUsers.query.filter( - sqlalchemy.and_( - models.ProjectUsers.user_id == user.user_id, - models.ProjectUsers.project_id == project.id, - ) - ).one_or_none() - assert project_user_before_addition is None - - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["researchuser"]).token(client), - query_string={"project": project_id}, - json=user_copy, - ) - assert response.status_code == http.HTTPStatus.FORBIDDEN - - project_user_after_addition = models.ProjectUsers.query.filter( - sqlalchemy.and_( - models.ProjectUsers.user_id == user.user_id, - models.ProjectUsers.project_id == project.id, - ) - ).one_or_none() - assert project_user_after_addition is None - - -# projectowner adds researchuser2 to projects[0] -def test_project_owner_can_add_existing_user_to_existing_project(client): - user_copy = existing_research_user_to_existing_project.copy() - project_id = user_copy.pop("project") - - project = models.Project.query.filter_by(public_id=project_id).one_or_none() - user = models.Email.query.filter_by( - email=existing_research_user_to_existing_project["email"] - ).one_or_none() - project_user_before_addition = models.ProjectUsers.query.filter( - sqlalchemy.and_( - models.ProjectUsers.user_id == user.user_id, - models.ProjectUsers.project_id == project.id, - ) - ).one_or_none() - assert project_user_before_addition is None - - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["projectowner"]).token(client), - query_string={"project": project_id}, - json=user_copy, - ) - assert response.status_code == http.HTTPStatus.OK - - project_user_after_addition = models.ProjectUsers.query.filter( - sqlalchemy.and_( - models.ProjectUsers.user_id == user.user_id, - models.ProjectUsers.project_id == project.id, - ) - ).one_or_none() - assert project_user_after_addition is not None - - -def test_add_existing_user_to_existing_project(client): - user_copy = existing_research_user_to_existing_project.copy() - project_id = user_copy.pop("project") - - project = models.Project.query.filter_by(public_id=project_id).one_or_none() - user = models.Email.query.filter_by( - email=existing_research_user_to_existing_project["email"] - ).one_or_none() - project_user_before_addition = models.ProjectUsers.query.filter( - sqlalchemy.and_( - models.ProjectUsers.user_id == user.user_id, - models.ProjectUsers.project_id == project.id, - ) - ).one_or_none() - assert project_user_before_addition is None - - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - query_string={"project": project_id}, - json=user_copy, - ) - assert response.status_code == http.HTTPStatus.OK - - project_user_after_addition = models.ProjectUsers.query.filter( - sqlalchemy.and_( - models.ProjectUsers.user_id == user.user_id, - models.ProjectUsers.project_id == project.id, - ) - ).one_or_none() - assert project_user_after_addition - - -def test_add_existing_user_to_existing_project_no_mail_flag(client): - "Test that an e-mail notification is not send when the --no-mail flag is used" - - user_copy = existing_research_user_to_existing_project.copy() - project_id = user_copy.pop("project") - new_status = {"new_status": "Available"} - user_copy["send_email"] = False - token = tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client) - - # make project available prior to test, otherwise an e-mail is never sent. - response = client.post( - tests.DDSEndpoint.PROJECT_STATUS, - headers=token, - query_string={"project": project_id}, - data=json.dumps(new_status), - content_type="application/json", - ) - - # Test mail sending is suppressed - - with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: - with unittest.mock.patch.object( - dds_web.api.user.AddUser, "compose_and_send_email_to_user" - ) as mock_mail_func: - print(user_copy) - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=token, - query_string={"project": project_id}, - data=json.dumps(user_copy), - content_type="application/json", - ) - # assert that no mail is being sent. - assert mock_mail_func.called == False - assert mock_mail_send.call_count == 0 - - assert response.status_code == http.HTTPStatus.OK - assert "An e-mail notification has not been sent." in response.json["message"] - - -def test_add_existing_user_to_existing_project_after_release(client): - user_copy = existing_research_user_to_existing_project.copy() - project_id = user_copy.pop("project") - - project = models.Project.query.filter_by(public_id=project_id).one_or_none() - user = models.Email.query.filter_by( - email=existing_research_user_to_existing_project["email"] - ).one_or_none() - project_user_before_addition = models.ProjectUsers.query.filter( - sqlalchemy.and_( - models.ProjectUsers.user_id == user.user_id, - models.ProjectUsers.project_id == project.id, - ) - ).one_or_none() - assert project_user_before_addition is None - - # release project - response = client.post( - tests.DDSEndpoint.PROJECT_STATUS, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - query_string={"project": project_id}, - json={"new_status": "Available"}, - ) - assert response.status_code == http.HTTPStatus.OK - assert project.current_status == "Available" - - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - query_string={"project": project_id}, - json=user_copy, - ) - assert response.status_code == http.HTTPStatus.OK - - project_user_after_addition = models.ProjectUsers.query.filter( - sqlalchemy.and_( - models.ProjectUsers.user_id == user.user_id, - models.ProjectUsers.project_id == project.id, - ) - ).one_or_none() - assert project_user_after_addition - - -def test_add_existing_user_to_nonexistent_proj(client): - user_copy = existing_research_user_to_nonexistent_proj.copy() - project = user_copy.pop("project") - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - query_string={"project": project}, - json=user_copy, - ) - assert response.status_code == http.HTTPStatus.BAD_REQUEST - - -def test_existing_user_change_ownership(client): - project = models.Project.query.filter_by( - public_id=change_owner_existing_user["project"] - ).one_or_none() - user = models.Email.query.filter_by(email=change_owner_existing_user["email"]).one_or_none() - project_user = models.ProjectUsers.query.filter( - sqlalchemy.and_( - models.ProjectUsers.user_id == user.user_id, - models.ProjectUsers.project_id == project.id, - ) - ).one_or_none() - - assert not project_user.owner - - user_new_owner_status = change_owner_existing_user.copy() - project = user_new_owner_status.pop("project") - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - query_string={"project": project}, - json=user_new_owner_status, - ) - - assert response.status_code == http.HTTPStatus.OK - - db.session.refresh(project_user) - - assert project_user.owner - - -def test_existing_user_change_ownership_same_permissions(client): - user_same_ownership = submit_with_same_ownership.copy() - project = user_same_ownership.pop("project") - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - query_string={"project": project}, - json=user_same_ownership, + json=new_super_admin, ) - assert response.status_code == http.HTTPStatus.FORBIDDEN - -def test_add_existing_user_with_unsuitable_role(client): - user_with_unsuitable_role = existing_research_user_to_existing_project.copy() - user_with_unsuitable_role["role"] = "Unit Admin" - project = user_with_unsuitable_role.pop("project") - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - query_string={"project": project}, - json=user_with_unsuitable_role, - ) assert response.status_code == http.HTTPStatus.FORBIDDEN - - -# Invite to project ########################################################### Invite to project # - - -def test_invite_with_project_by_unituser(client): - "Test that a new invite including a project can be created" - - project = existing_project - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - query_string={"project": project}, - json=first_new_user, - ) - assert response.status_code == http.HTTPStatus.OK - - invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() - assert invited_user - assert invited_user.email == first_new_user["email"] - assert invited_user.role == first_new_user["role"] - - assert invited_user.nonce is not None - assert invited_user.public_key is not None - assert invited_user.private_key is not None - - project_invite_keys = invited_user.project_invite_keys - assert len(project_invite_keys) == 1 - assert project_invite_keys[0].project.public_id == project - assert not project_invite_keys[0].owner - - -def test_add_project_to_existing_invite_by_unituser(client): - "Test that a project can be associated with an existing invite" - - # Create invite upfront - - project = existing_project - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - json=first_new_user, - ) - assert response.status_code == http.HTTPStatus.OK - - invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() - - # Check that the invite has no projects yet - - assert invited_user - assert len(invited_user.project_invite_keys) == 0 - - # Add project to existing invite - - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - query_string={"project": project}, - json=first_new_user, - ) - - assert response.status_code == http.HTTPStatus.OK - - # Check that the invite has now a project association - project_invite_keys = invited_user.project_invite_keys - assert len(project_invite_keys) == 1 - assert project_invite_keys[0].project.public_id == project - assert not project_invite_keys[0].owner - - -def test_update_project_to_existing_invite_by_unituser(client): - "Test that project ownership can be updated for an existing invite" - - # Create Invite upfront - - project = existing_project - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - query_string={"project": project}, - json=first_new_user, - ) - assert response.status_code == http.HTTPStatus.OK - - project_obj = models.Project.query.filter_by(public_id=existing_project).one_or_none() - invite_obj = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() - - project_invite = models.ProjectInviteKeys.query.filter( - sqlalchemy.and_( - models.ProjectInviteKeys.invite_id == invite_obj.id, - models.ProjectUserKeys.project_id == project_obj.id, - ) - ).one_or_none() - - assert project_invite - assert not project_invite.owner - - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - query_string={"project": project}, - json=first_new_owner, - ) - - assert response.status_code == http.HTTPStatus.OK - - db.session.refresh(project_invite) - - assert project_invite.owner - - -def test_invited_as_owner_and_researcher_to_different_project(client): - "Test that an invite can be owner of one project and researcher of another" - - # Create Invite upfront as owner - - project = existing_project - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - query_string={"project": project}, - json=first_new_owner, - ) - assert response.status_code == http.HTTPStatus.OK - - # Perform second invite as researcher - project2 = existing_project_2 - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - query_string={"project": project2}, - json=first_new_user, - ) - assert response.status_code == http.HTTPStatus.OK - - project_obj_owner = models.Project.query.filter_by(public_id=existing_project).one_or_none() - project_obj_not_owner = models.Project.query.filter_by( - public_id=existing_project_2 - ).one_or_none() - - invite_obj = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() - - project_invite_owner = models.ProjectInviteKeys.query.filter( - sqlalchemy.and_( - models.ProjectInviteKeys.invite_id == invite_obj.id, - models.ProjectInviteKeys.project_id == project_obj_owner.id, - ) - ).one_or_none() - - assert project_invite_owner - assert project_invite_owner.owner - - project_invite_not_owner = models.ProjectInviteKeys.query.filter( - sqlalchemy.and_( - models.ProjectInviteKeys.invite_id == invite_obj.id, - models.ProjectInviteKeys.project_id == project_obj_not_owner.id, - ) - ).one_or_none() - - assert project_invite_not_owner - assert not project_invite_not_owner.owner - - # Owner or not should not be stored on the invite - assert invite_obj.role == "Researcher" - - -def test_invite_to_project_by_project_owner(client): - "Test that a project owner can invite to its project" - - project = existing_project - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["projectowner"]).token(client), - query_string={"project": project}, - json=first_new_user, - ) - assert response.status_code == http.HTTPStatus.OK - - invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() - assert invited_user - assert invited_user.email == first_new_user["email"] - assert invited_user.role == first_new_user["role"] - - assert invited_user.nonce is not None - assert invited_user.public_key is not None - assert invited_user.private_key is not None - - project_invite_keys = invited_user.project_invite_keys - assert len(project_invite_keys) == 1 - assert project_invite_keys[0].project.public_id == project - assert not project_invite_keys[0].owner - - -def test_add_anyuser_to_project_with_superadmin(client): - """Super admins cannot invite to project.""" - project = existing_project - for x in [first_new_user, first_new_owner, new_unit_user, new_unit_admin]: - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client), - query_string={"project": project}, - json=x, - ) - assert response.status_code == http.HTTPStatus.BAD_REQUEST - - # An invite should not have been created - invited_user = models.Invite.query.filter_by(email=x["email"]).one_or_none() - assert not invited_user - - -def test_add_unituser_and_admin_no_unit_with_superadmin(client): - """A super admin needs to specify a unit to be able to invite unit users.""" - project = existing_project - for x in [new_unit_user, new_unit_admin]: - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client), - json=x, - ) - - assert "You need to specify a unit" in response.json["message"] - assert response.status_code == http.HTTPStatus.BAD_REQUEST - - invited_user = models.Invite.query.filter_by(email=x["email"]).one_or_none() - assert not invited_user - - -def test_add_researchuser_project_no_access_unit_admin_and_personnel(client): - """A unit admin and personnel should not be able to give access to a project - which they themselves have lost access to.""" - # Make sure the project exists - project = models.Project.query.filter_by(public_id=existing_project).one_or_none() - assert project - - for inviter in ["unitadmin", "unituser", "projectowner"]: - # Check that the unit admin has access to the project first - project_user_key = models.ProjectUserKeys.query.filter_by( - user_id=inviter, project_id=project.id - ).one_or_none() - assert project_user_key - - # Remove the project access (for test) - db.session.delete(project_user_key) - - # Make sure the project access does not exist now - project_user_key = models.ProjectUserKeys.query.filter_by( - user_id=inviter, project_id=project.id - ).one_or_none() - assert not project_user_key - - for x in [first_new_user, first_new_owner]: - # Make sure there is no ongoing invite - invited_user_before = models.Invite.query.filter_by(email=x["email"]).one_or_none() - if invited_user_before: - db.session.delete(invited_user_before) - invited_user_before = models.Invite.query.filter_by(email=x["email"]).one_or_none() - assert not invited_user_before - - # Attempt invite - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS[inviter]).token(client), - json=x, - query_string={"project": project.public_id}, - ) - - # The invite should still be done, but they can't invite to a project - invited_user = models.Invite.query.filter_by(email=x["email"]).one_or_none() - assert invited_user - - # Make sure there are no project v - assert not invited_user.project_invite_keys - - # There should be an error message - assert response.status_code == http.HTTPStatus.BAD_REQUEST - assert "The user could not be added to the project(s)" in response.json["message"] - - # Verify ok error messages - assert "errors" in response.json - assert project.public_id in response.json["errors"] - assert ( - "You do not have access to the specified project." - in response.json["errors"][project.public_id] - ) - - -# Invite without email -def test_invite_without_email(client): - """The email is required.""" - user_no_email = first_new_user.copy() - user_no_email.pop("email") - - for inviter in ["superadmin", "unitadmin", "unituser"]: - # Attempt invite - response = client.post( - tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS[inviter]).token(client), - json=user_no_email, - # query_string={"project": existing_project}, - ) - - # There should be an error message - assert response.status_code == http.HTTPStatus.BAD_REQUEST - assert "Email address required to add or invite." in response.json["message"] + assert "The user does not have the necessary permissions." in response.json["message"] From f984e96d7d567896ff3e4bf5a9c5672ea597b497 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Sun, 13 Mar 2022 11:32:04 +0100 Subject: [PATCH 047/293] test invite superadmin and unit admin as unit personnel --- tests/test_user_add.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_user_add.py b/tests/test_user_add.py index 6b0f640fc..1db6ab575 100644 --- a/tests/test_user_add.py +++ b/tests/test_user_add.py @@ -832,3 +832,18 @@ def test_invite_superadmin_as_unitadmin(client): assert response.status_code == http.HTTPStatus.FORBIDDEN assert "The user does not have the necessary permissions." in response.json["message"] + + +# Invite super admin with unit admin +def test_invite_superadmin_and_unitadmin_as_unitpersonnel(client): + """A unit personnel cannot invite a superadmin or unit admin""" + for invitee in ["superadmin", "unitadmin"]: + # Attempt invite + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + json=new_super_admin, + ) + + assert response.status_code == http.HTTPStatus.FORBIDDEN + assert "The user does not have the necessary permissions." in response.json["message"] From 4b48d76acc7b0fc6ab1c2564614e3aa634ef14f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Sun, 13 Mar 2022 11:36:25 +0100 Subject: [PATCH 048/293] test invite other roles as project owner --- tests/test_user_add.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/test_user_add.py b/tests/test_user_add.py index 1db6ab575..34aaa25a4 100644 --- a/tests/test_user_add.py +++ b/tests/test_user_add.py @@ -834,7 +834,7 @@ def test_invite_superadmin_as_unitadmin(client): assert "The user does not have the necessary permissions." in response.json["message"] -# Invite super admin with unit admin +# Invite super admin and unit admin with unit personnel def test_invite_superadmin_and_unitadmin_as_unitpersonnel(client): """A unit personnel cannot invite a superadmin or unit admin""" for invitee in ["superadmin", "unitadmin"]: @@ -847,3 +847,19 @@ def test_invite_superadmin_and_unitadmin_as_unitpersonnel(client): assert response.status_code == http.HTTPStatus.FORBIDDEN assert "The user does not have the necessary permissions." in response.json["message"] + + +# Invite super admin, unit admin or unit personnel +def test_invite_superadmin_and_unitadmin_and_unitpersonnel_as_projectowner(client): + """A project owner cannot invite a superadmin or unit admin or unit personnel.""" + for invitee in ["superadmin", "unitadmin", "unituser"]: + # Attempt invite + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["projectowner"]).token(client), + json=new_super_admin, + query_string={"project": existing_project}, + ) + + assert response.status_code == http.HTTPStatus.FORBIDDEN + assert "The user does not have the necessary permissions." in response.json["message"] From e4679c84b49017b0e567276ba1fb8506d196a73e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Sun, 13 Mar 2022 11:41:19 +0100 Subject: [PATCH 049/293] fixed tests --- tests/test_user_add.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/tests/test_user_add.py b/tests/test_user_add.py index 34aaa25a4..573d545b7 100644 --- a/tests/test_user_add.py +++ b/tests/test_user_add.py @@ -837,12 +837,12 @@ def test_invite_superadmin_as_unitadmin(client): # Invite super admin and unit admin with unit personnel def test_invite_superadmin_and_unitadmin_as_unitpersonnel(client): """A unit personnel cannot invite a superadmin or unit admin""" - for invitee in ["superadmin", "unitadmin"]: + for invitee in [new_super_admin, new_unit_admin]: # Attempt invite response = client.post( tests.DDSEndpoint.USER_ADD, headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), - json=new_super_admin, + json=invitee, ) assert response.status_code == http.HTTPStatus.FORBIDDEN @@ -852,12 +852,34 @@ def test_invite_superadmin_and_unitadmin_as_unitpersonnel(client): # Invite super admin, unit admin or unit personnel def test_invite_superadmin_and_unitadmin_and_unitpersonnel_as_projectowner(client): """A project owner cannot invite a superadmin or unit admin or unit personnel.""" - for invitee in ["superadmin", "unitadmin", "unituser"]: + for invitee in [new_super_admin, new_unit_admin, new_unit_user]: # Attempt invite response = client.post( tests.DDSEndpoint.USER_ADD, headers=tests.UserAuth(tests.USER_CREDENTIALS["projectowner"]).token(client), - json=new_super_admin, + json=invitee, + query_string={"project": existing_project}, + ) + + assert response.status_code == http.HTTPStatus.FORBIDDEN + assert "The user does not have the necessary permissions." in response.json["message"] + + +# Invite as researcher +def test_invite_as_researcher(client): + """A researcher cannot invite.""" + for invitee in [ + new_super_admin, + new_unit_admin, + new_unit_user, + first_new_owner, + first_new_user, + ]: + # Attempt invite + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["projectowner"]).token(client), + json=invitee, query_string={"project": existing_project}, ) From a1170876f857dbd4e5b3225005d2326fdef11476 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Sun, 13 Mar 2022 11:49:27 +0100 Subject: [PATCH 050/293] test invite with invalid unit --- tests/test_user_add.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/tests/test_user_add.py b/tests/test_user_add.py index 573d545b7..d32048d6e 100644 --- a/tests/test_user_add.py +++ b/tests/test_user_add.py @@ -865,23 +865,18 @@ def test_invite_superadmin_and_unitadmin_and_unitpersonnel_as_projectowner(clien assert "The user does not have the necessary permissions." in response.json["message"] -# Invite as researcher -def test_invite_as_researcher(client): - """A researcher cannot invite.""" - for invitee in [ - new_super_admin, - new_unit_admin, - new_unit_user, - first_new_owner, - first_new_user, - ]: +def test_invite_unituser_as_superadmin_incorrect_unit(client): + """A valid unit is required for super admins to invite unit users.""" + for invitee in [new_unit_admin, new_unit_user]: + invite_with_invalid_unit = invitee.copy() + invite_with_invalid_unit["unit"] = "invalidunit" + # Attempt invite response = client.post( tests.DDSEndpoint.USER_ADD, - headers=tests.UserAuth(tests.USER_CREDENTIALS["projectowner"]).token(client), - json=invitee, - query_string={"project": existing_project}, + headers=tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client), + json=invite_with_invalid_unit, ) - assert response.status_code == http.HTTPStatus.FORBIDDEN - assert "The user does not have the necessary permissions." in response.json["message"] + assert response.status_code == http.HTTPStatus.BAD_REQUEST + assert "Invalid unit publid id." in response.json["message"] From 226eba9a0d976a9de7754d35e374270a57284be6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Sun, 13 Mar 2022 11:57:29 +0100 Subject: [PATCH 051/293] test with valid unit --- tests/test_user_add.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_user_add.py b/tests/test_user_add.py index d32048d6e..03a1ab789 100644 --- a/tests/test_user_add.py +++ b/tests/test_user_add.py @@ -880,3 +880,25 @@ def test_invite_unituser_as_superadmin_incorrect_unit(client): assert response.status_code == http.HTTPStatus.BAD_REQUEST assert "Invalid unit publid id." in response.json["message"] + + +def test_invite_unituser_with_valid_unit_as_superadmin(client): + """A unit user should be invited if the super admin provides a valid unit.""" + for invitee in [new_unit_admin, new_unit_user]: + valid_unit = models.Unit.query.filter_by(name="Unit 1").one_or_none() + assert valid_unit + + invite_with_valid_unit = invitee.copy() + invite_with_valid_unit["unit"] = valid_unit.public_id + + # Attempt invite + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client), + json=invite_with_valid_unit, + ) + + new_invite = models.Invite.query.filter_by( + email=invite_with_valid_unit["email"] + ).one_or_none() + assert new_invite From f4b1520e8ed8b1220bd2f6cf6f62206dfab26dc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Sun, 13 Mar 2022 12:04:07 +0100 Subject: [PATCH 052/293] push the old tests too --- tests/test_user_add.py | 1536 ++++++++++++++++++++-------------------- 1 file changed, 768 insertions(+), 768 deletions(-) diff --git a/tests/test_user_add.py b/tests/test_user_add.py index 03a1ab789..fcd645719 100644 --- a/tests/test_user_add.py +++ b/tests/test_user_add.py @@ -44,780 +44,780 @@ "project": "second_public_project_id", } -# # Inviting Users ################################################################# Inviting Users # -# def test_add_user_with_researcher(client): -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["researchuser"]).token(client), -# json=first_new_user, -# ) -# assert response.status_code == http.HTTPStatus.FORBIDDEN -# invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() -# assert invited_user is None - - -# def test_add_user_with_unituser_no_role(client): -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), -# json=first_new_email, -# ) -# assert response.status_code == http.HTTPStatus.BAD_REQUEST -# invited_user = models.Invite.query.filter_by(email=first_new_email["email"]).one_or_none() -# assert invited_user is None - - -# def test_add_user_with_unitadmin_with_extraargs(client): -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), -# json=first_new_user_extra_args, -# ) -# assert response.status_code == http.HTTPStatus.OK -# invited_user = models.Invite.query.filter_by( -# email=first_new_user_extra_args["email"] -# ).one_or_none() -# assert invited_user - - -# def test_add_user_with_unitadmin_and_invalid_role(client): -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), -# json=first_new_user_invalid_role, -# ) -# assert response.status_code == http.HTTPStatus.BAD_REQUEST -# invited_user = models.Invite.query.filter_by( -# email=first_new_user_invalid_role["email"] -# ).one_or_none() -# assert invited_user is None - - -# def test_add_user_with_unitadmin_and_invalid_email(client): -# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), -# json=first_new_user_invalid_email, -# ) -# assert response.status_code == http.HTTPStatus.BAD_REQUEST -# # An email is always sent when receiving the partial token -# mock_mail_send.assert_called_once() - -# invited_user = models.Invite.query.filter_by( -# email=first_new_user_invalid_email["email"] -# ).one_or_none() -# assert invited_user is None - - -# def test_add_user_with_unitadmin(client): -# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: -# token = tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client) -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=token, -# json=first_new_user, -# ) -# # One mail sent for partial token and one for the invite -# assert mock_mail_send.call_count == 2 - -# assert response.status_code == http.HTTPStatus.OK - -# invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() -# assert invited_user -# assert invited_user.email == first_new_user["email"] -# assert invited_user.role == first_new_user["role"] - -# assert invited_user.nonce is not None -# assert invited_user.public_key is not None -# assert invited_user.private_key is not None -# assert invited_user.project_invite_keys == [] - -# # Repeating the invite should not send a new invite: -# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=token, -# json=first_new_user, -# ) -# # No new mail should be sent for the token and neither for an invite -# assert mock_mail_send.call_count == 0 -# assert response.status_code == http.HTTPStatus.BAD_REQUEST -# message = response.json.get("message") -# assert "user was already added to the system" in message - - -# def test_add_unit_user_with_unitadmin(client): - -# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: -# token = tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client) -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=token, -# json=new_unit_user, -# ) -# # One mail sent for partial token and one for the invite -# assert mock_mail_send.call_count == 2 - -# assert response.status_code == http.HTTPStatus.OK - -# invited_user = models.Invite.query.filter_by(email=new_unit_user["email"]).one_or_none() -# assert invited_user -# assert invited_user.email == new_unit_user["email"] -# assert invited_user.role == new_unit_user["role"] - -# assert invited_user.nonce is not None -# assert invited_user.public_key is not None -# assert invited_user.private_key is not None - -# project_invite_keys = invited_user.project_invite_keys -# number_of_asserted_projects = 0 -# for project_invite_key in project_invite_keys: -# if ( -# project_invite_key.project.public_id == "public_project_id" -# or project_invite_key.project.public_id == "unused_project_id" -# or project_invite_key.project.public_id == "restricted_project_id" -# or project_invite_key.project.public_id == "second_public_project_id" -# or project_invite_key.project.public_id == "file_testing_project" -# ): -# number_of_asserted_projects += 1 -# assert len(project_invite_keys) == number_of_asserted_projects -# assert len(project_invite_keys) == len(invited_user.unit.projects) -# assert len(project_invite_keys) == 5 - -# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=token, -# json=new_unit_user, -# ) -# # No new mail should be sent for the token and neither for an invite -# assert mock_mail_send.call_count == 0 - -# assert response.status_code == http.HTTPStatus.BAD_REQUEST -# message = response.json.get("message") -# assert "user was already added to the system" in message - - -# def test_add_user_with_superadmin(client): -# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: -# token = tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client) -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=token, -# json=first_new_user, -# ) -# # One mail sent for partial token and one for the invite -# assert mock_mail_send.call_count == 2 - -# assert response.status_code == http.HTTPStatus.OK - -# invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() -# assert invited_user -# assert invited_user.email == first_new_user["email"] -# assert invited_user.role == first_new_user["role"] - -# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=token, -# json=first_new_user, -# ) -# # No new mail should be sent for the token and neither for an invite -# assert mock_mail_send.call_count == 0 - -# assert response.status_code == http.HTTPStatus.BAD_REQUEST -# message = response.json.get("message") -# assert "user was already added to the system" in message - - -# def test_add_user_existing_email_no_project(client): -# invited_user = models.Invite.query.filter_by( -# email=existing_invite["email"], role=existing_invite["role"] -# ).one_or_none() -# assert invited_user -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), -# json=existing_invite, -# ) -# assert response.status_code == http.HTTPStatus.BAD_REQUEST - - -# def test_add_unitadmin_user_with_unitpersonnel_permission_denied(client): -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# json=new_unit_admin, -# ) -# assert response.status_code == http.HTTPStatus.FORBIDDEN - -# invited_user = models.Invite.query.filter_by(email=new_unit_admin["email"]).one_or_none() -# assert invited_user is None - - -# # Add existing users to projects ################################# Add existing users to projects # -# def test_add_existing_user_without_project(client): -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# json=existing_research_user, -# ) -# assert response.status_code == http.HTTPStatus.BAD_REQUEST - - -# def test_research_user_cannot_add_existing_user_to_existing_project(client): -# user_copy = existing_research_user_to_existing_project.copy() -# project_id = user_copy.pop("project") - -# project = models.Project.query.filter_by(public_id=project_id).one_or_none() -# user = models.Email.query.filter_by( -# email=existing_research_user_to_existing_project["email"] -# ).one_or_none() -# project_user_before_addition = models.ProjectUsers.query.filter( -# sqlalchemy.and_( -# models.ProjectUsers.user_id == user.user_id, -# models.ProjectUsers.project_id == project.id, -# ) -# ).one_or_none() -# assert project_user_before_addition is None - -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["researchuser"]).token(client), -# query_string={"project": project_id}, -# json=user_copy, -# ) -# assert response.status_code == http.HTTPStatus.FORBIDDEN - -# project_user_after_addition = models.ProjectUsers.query.filter( -# sqlalchemy.and_( -# models.ProjectUsers.user_id == user.user_id, -# models.ProjectUsers.project_id == project.id, -# ) -# ).one_or_none() -# assert project_user_after_addition is None - - -# # projectowner adds researchuser2 to projects[0] -# def test_project_owner_can_add_existing_user_to_existing_project(client): -# user_copy = existing_research_user_to_existing_project.copy() -# project_id = user_copy.pop("project") - -# project = models.Project.query.filter_by(public_id=project_id).one_or_none() -# user = models.Email.query.filter_by( -# email=existing_research_user_to_existing_project["email"] -# ).one_or_none() -# project_user_before_addition = models.ProjectUsers.query.filter( -# sqlalchemy.and_( -# models.ProjectUsers.user_id == user.user_id, -# models.ProjectUsers.project_id == project.id, -# ) -# ).one_or_none() -# assert project_user_before_addition is None - -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["projectowner"]).token(client), -# query_string={"project": project_id}, -# json=user_copy, -# ) -# assert response.status_code == http.HTTPStatus.OK - -# project_user_after_addition = models.ProjectUsers.query.filter( -# sqlalchemy.and_( -# models.ProjectUsers.user_id == user.user_id, -# models.ProjectUsers.project_id == project.id, -# ) -# ).one_or_none() -# assert project_user_after_addition is not None - - -# def test_add_existing_user_to_existing_project(client): -# user_copy = existing_research_user_to_existing_project.copy() -# project_id = user_copy.pop("project") - -# project = models.Project.query.filter_by(public_id=project_id).one_or_none() -# user = models.Email.query.filter_by( -# email=existing_research_user_to_existing_project["email"] -# ).one_or_none() -# project_user_before_addition = models.ProjectUsers.query.filter( -# sqlalchemy.and_( -# models.ProjectUsers.user_id == user.user_id, -# models.ProjectUsers.project_id == project.id, -# ) -# ).one_or_none() -# assert project_user_before_addition is None - -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# query_string={"project": project_id}, -# json=user_copy, -# ) -# assert response.status_code == http.HTTPStatus.OK - -# project_user_after_addition = models.ProjectUsers.query.filter( -# sqlalchemy.and_( -# models.ProjectUsers.user_id == user.user_id, -# models.ProjectUsers.project_id == project.id, -# ) -# ).one_or_none() -# assert project_user_after_addition - - -# def test_add_existing_user_to_existing_project_no_mail_flag(client): -# "Test that an e-mail notification is not send when the --no-mail flag is used" - -# user_copy = existing_research_user_to_existing_project.copy() -# project_id = user_copy.pop("project") -# new_status = {"new_status": "Available"} -# user_copy["send_email"] = False -# token = tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client) - -# # make project available prior to test, otherwise an e-mail is never sent. -# response = client.post( -# tests.DDSEndpoint.PROJECT_STATUS, -# headers=token, -# query_string={"project": project_id}, -# data=json.dumps(new_status), -# content_type="application/json", -# ) - -# # Test mail sending is suppressed - -# with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: -# with unittest.mock.patch.object( -# dds_web.api.user.AddUser, "compose_and_send_email_to_user" -# ) as mock_mail_func: -# print(user_copy) -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=token, -# query_string={"project": project_id}, -# data=json.dumps(user_copy), -# content_type="application/json", -# ) -# # assert that no mail is being sent. -# assert mock_mail_func.called == False -# assert mock_mail_send.call_count == 0 - -# assert response.status_code == http.HTTPStatus.OK -# assert "An e-mail notification has not been sent." in response.json["message"] - - -# def test_add_existing_user_to_existing_project_after_release(client): -# user_copy = existing_research_user_to_existing_project.copy() -# project_id = user_copy.pop("project") - -# project = models.Project.query.filter_by(public_id=project_id).one_or_none() -# user = models.Email.query.filter_by( -# email=existing_research_user_to_existing_project["email"] -# ).one_or_none() -# project_user_before_addition = models.ProjectUsers.query.filter( -# sqlalchemy.and_( -# models.ProjectUsers.user_id == user.user_id, -# models.ProjectUsers.project_id == project.id, -# ) -# ).one_or_none() -# assert project_user_before_addition is None - -# # release project -# response = client.post( -# tests.DDSEndpoint.PROJECT_STATUS, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# query_string={"project": project_id}, -# json={"new_status": "Available"}, -# ) -# assert response.status_code == http.HTTPStatus.OK -# assert project.current_status == "Available" - -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# query_string={"project": project_id}, -# json=user_copy, -# ) -# assert response.status_code == http.HTTPStatus.OK - -# project_user_after_addition = models.ProjectUsers.query.filter( -# sqlalchemy.and_( -# models.ProjectUsers.user_id == user.user_id, -# models.ProjectUsers.project_id == project.id, -# ) -# ).one_or_none() -# assert project_user_after_addition - - -# def test_add_existing_user_to_nonexistent_proj(client): -# user_copy = existing_research_user_to_nonexistent_proj.copy() -# project = user_copy.pop("project") -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# query_string={"project": project}, -# json=user_copy, -# ) -# assert response.status_code == http.HTTPStatus.BAD_REQUEST - - -# def test_existing_user_change_ownership(client): -# project = models.Project.query.filter_by( -# public_id=change_owner_existing_user["project"] -# ).one_or_none() -# user = models.Email.query.filter_by(email=change_owner_existing_user["email"]).one_or_none() -# project_user = models.ProjectUsers.query.filter( -# sqlalchemy.and_( -# models.ProjectUsers.user_id == user.user_id, -# models.ProjectUsers.project_id == project.id, -# ) -# ).one_or_none() - -# assert not project_user.owner - -# user_new_owner_status = change_owner_existing_user.copy() -# project = user_new_owner_status.pop("project") -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# query_string={"project": project}, -# json=user_new_owner_status, -# ) - -# assert response.status_code == http.HTTPStatus.OK - -# db.session.refresh(project_user) - -# assert project_user.owner - - -# def test_existing_user_change_ownership_same_permissions(client): -# user_same_ownership = submit_with_same_ownership.copy() -# project = user_same_ownership.pop("project") -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# query_string={"project": project}, -# json=user_same_ownership, -# ) -# assert response.status_code == http.HTTPStatus.FORBIDDEN - - -# def test_add_existing_user_with_unsuitable_role(client): -# user_with_unsuitable_role = existing_research_user_to_existing_project.copy() -# user_with_unsuitable_role["role"] = "Unit Admin" -# project = user_with_unsuitable_role.pop("project") -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# query_string={"project": project}, -# json=user_with_unsuitable_role, -# ) -# assert response.status_code == http.HTTPStatus.FORBIDDEN - - -# # Invite to project ########################################################### Invite to project # - - -# def test_invite_with_project_by_unituser(client): -# "Test that a new invite including a project can be created" - -# project = existing_project -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# query_string={"project": project}, -# json=first_new_user, -# ) -# assert response.status_code == http.HTTPStatus.OK - -# invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() -# assert invited_user -# assert invited_user.email == first_new_user["email"] -# assert invited_user.role == first_new_user["role"] - -# assert invited_user.nonce is not None -# assert invited_user.public_key is not None -# assert invited_user.private_key is not None - -# project_invite_keys = invited_user.project_invite_keys -# assert len(project_invite_keys) == 1 -# assert project_invite_keys[0].project.public_id == project -# assert not project_invite_keys[0].owner - - -# def test_add_project_to_existing_invite_by_unituser(client): -# "Test that a project can be associated with an existing invite" - -# # Create invite upfront - -# project = existing_project -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# json=first_new_user, -# ) -# assert response.status_code == http.HTTPStatus.OK - -# invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() - -# # Check that the invite has no projects yet - -# assert invited_user -# assert len(invited_user.project_invite_keys) == 0 - -# # Add project to existing invite - -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# query_string={"project": project}, -# json=first_new_user, -# ) - -# assert response.status_code == http.HTTPStatus.OK - -# # Check that the invite has now a project association -# project_invite_keys = invited_user.project_invite_keys -# assert len(project_invite_keys) == 1 -# assert project_invite_keys[0].project.public_id == project -# assert not project_invite_keys[0].owner +# Inviting Users ################################################################# Inviting Users # +def test_add_user_with_researcher(client): + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["researchuser"]).token(client), + json=first_new_user, + ) + assert response.status_code == http.HTTPStatus.FORBIDDEN + invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() + assert invited_user is None + + +def test_add_user_with_unituser_no_role(client): + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), + json=first_new_email, + ) + assert response.status_code == http.HTTPStatus.BAD_REQUEST + invited_user = models.Invite.query.filter_by(email=first_new_email["email"]).one_or_none() + assert invited_user is None + + +def test_add_user_with_unitadmin_with_extraargs(client): + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), + json=first_new_user_extra_args, + ) + assert response.status_code == http.HTTPStatus.OK + invited_user = models.Invite.query.filter_by( + email=first_new_user_extra_args["email"] + ).one_or_none() + assert invited_user + + +def test_add_user_with_unitadmin_and_invalid_role(client): + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), + json=first_new_user_invalid_role, + ) + assert response.status_code == http.HTTPStatus.BAD_REQUEST + invited_user = models.Invite.query.filter_by( + email=first_new_user_invalid_role["email"] + ).one_or_none() + assert invited_user is None + + +def test_add_user_with_unitadmin_and_invalid_email(client): + with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), + json=first_new_user_invalid_email, + ) + assert response.status_code == http.HTTPStatus.BAD_REQUEST + # An email is always sent when receiving the partial token + mock_mail_send.assert_called_once() + + invited_user = models.Invite.query.filter_by( + email=first_new_user_invalid_email["email"] + ).one_or_none() + assert invited_user is None + + +def test_add_user_with_unitadmin(client): + with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: + token = tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client) + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=token, + json=first_new_user, + ) + # One mail sent for partial token and one for the invite + assert mock_mail_send.call_count == 2 + + assert response.status_code == http.HTTPStatus.OK + + invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() + assert invited_user + assert invited_user.email == first_new_user["email"] + assert invited_user.role == first_new_user["role"] + + assert invited_user.nonce is not None + assert invited_user.public_key is not None + assert invited_user.private_key is not None + assert invited_user.project_invite_keys == [] + + # Repeating the invite should not send a new invite: + with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=token, + json=first_new_user, + ) + # No new mail should be sent for the token and neither for an invite + assert mock_mail_send.call_count == 0 + assert response.status_code == http.HTTPStatus.BAD_REQUEST + message = response.json.get("message") + assert "user was already added to the system" in message + + +def test_add_unit_user_with_unitadmin(client): + + with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: + token = tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client) + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=token, + json=new_unit_user, + ) + # One mail sent for partial token and one for the invite + assert mock_mail_send.call_count == 2 + + assert response.status_code == http.HTTPStatus.OK + + invited_user = models.Invite.query.filter_by(email=new_unit_user["email"]).one_or_none() + assert invited_user + assert invited_user.email == new_unit_user["email"] + assert invited_user.role == new_unit_user["role"] + + assert invited_user.nonce is not None + assert invited_user.public_key is not None + assert invited_user.private_key is not None + + project_invite_keys = invited_user.project_invite_keys + number_of_asserted_projects = 0 + for project_invite_key in project_invite_keys: + if ( + project_invite_key.project.public_id == "public_project_id" + or project_invite_key.project.public_id == "unused_project_id" + or project_invite_key.project.public_id == "restricted_project_id" + or project_invite_key.project.public_id == "second_public_project_id" + or project_invite_key.project.public_id == "file_testing_project" + ): + number_of_asserted_projects += 1 + assert len(project_invite_keys) == number_of_asserted_projects + assert len(project_invite_keys) == len(invited_user.unit.projects) + assert len(project_invite_keys) == 5 + + with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=token, + json=new_unit_user, + ) + # No new mail should be sent for the token and neither for an invite + assert mock_mail_send.call_count == 0 + + assert response.status_code == http.HTTPStatus.BAD_REQUEST + message = response.json.get("message") + assert "user was already added to the system" in message + + +def test_add_user_with_superadmin(client): + with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: + token = tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client) + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=token, + json=first_new_user, + ) + # One mail sent for partial token and one for the invite + assert mock_mail_send.call_count == 2 + + assert response.status_code == http.HTTPStatus.OK + + invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() + assert invited_user + assert invited_user.email == first_new_user["email"] + assert invited_user.role == first_new_user["role"] + + with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=token, + json=first_new_user, + ) + # No new mail should be sent for the token and neither for an invite + assert mock_mail_send.call_count == 0 + + assert response.status_code == http.HTTPStatus.BAD_REQUEST + message = response.json.get("message") + assert "user was already added to the system" in message + + +def test_add_user_existing_email_no_project(client): + invited_user = models.Invite.query.filter_by( + email=existing_invite["email"], role=existing_invite["role"] + ).one_or_none() + assert invited_user + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), + json=existing_invite, + ) + assert response.status_code == http.HTTPStatus.BAD_REQUEST + + +def test_add_unitadmin_user_with_unitpersonnel_permission_denied(client): + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + json=new_unit_admin, + ) + assert response.status_code == http.HTTPStatus.FORBIDDEN + + invited_user = models.Invite.query.filter_by(email=new_unit_admin["email"]).one_or_none() + assert invited_user is None + + +# Add existing users to projects ################################# Add existing users to projects # +def test_add_existing_user_without_project(client): + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + json=existing_research_user, + ) + assert response.status_code == http.HTTPStatus.BAD_REQUEST + + +def test_research_user_cannot_add_existing_user_to_existing_project(client): + user_copy = existing_research_user_to_existing_project.copy() + project_id = user_copy.pop("project") + + project = models.Project.query.filter_by(public_id=project_id).one_or_none() + user = models.Email.query.filter_by( + email=existing_research_user_to_existing_project["email"] + ).one_or_none() + project_user_before_addition = models.ProjectUsers.query.filter( + sqlalchemy.and_( + models.ProjectUsers.user_id == user.user_id, + models.ProjectUsers.project_id == project.id, + ) + ).one_or_none() + assert project_user_before_addition is None + + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["researchuser"]).token(client), + query_string={"project": project_id}, + json=user_copy, + ) + assert response.status_code == http.HTTPStatus.FORBIDDEN + + project_user_after_addition = models.ProjectUsers.query.filter( + sqlalchemy.and_( + models.ProjectUsers.user_id == user.user_id, + models.ProjectUsers.project_id == project.id, + ) + ).one_or_none() + assert project_user_after_addition is None + + +# projectowner adds researchuser2 to projects[0] +def test_project_owner_can_add_existing_user_to_existing_project(client): + user_copy = existing_research_user_to_existing_project.copy() + project_id = user_copy.pop("project") + + project = models.Project.query.filter_by(public_id=project_id).one_or_none() + user = models.Email.query.filter_by( + email=existing_research_user_to_existing_project["email"] + ).one_or_none() + project_user_before_addition = models.ProjectUsers.query.filter( + sqlalchemy.and_( + models.ProjectUsers.user_id == user.user_id, + models.ProjectUsers.project_id == project.id, + ) + ).one_or_none() + assert project_user_before_addition is None + + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["projectowner"]).token(client), + query_string={"project": project_id}, + json=user_copy, + ) + assert response.status_code == http.HTTPStatus.OK + + project_user_after_addition = models.ProjectUsers.query.filter( + sqlalchemy.and_( + models.ProjectUsers.user_id == user.user_id, + models.ProjectUsers.project_id == project.id, + ) + ).one_or_none() + assert project_user_after_addition is not None + + +def test_add_existing_user_to_existing_project(client): + user_copy = existing_research_user_to_existing_project.copy() + project_id = user_copy.pop("project") + + project = models.Project.query.filter_by(public_id=project_id).one_or_none() + user = models.Email.query.filter_by( + email=existing_research_user_to_existing_project["email"] + ).one_or_none() + project_user_before_addition = models.ProjectUsers.query.filter( + sqlalchemy.and_( + models.ProjectUsers.user_id == user.user_id, + models.ProjectUsers.project_id == project.id, + ) + ).one_or_none() + assert project_user_before_addition is None + + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + query_string={"project": project_id}, + json=user_copy, + ) + assert response.status_code == http.HTTPStatus.OK + + project_user_after_addition = models.ProjectUsers.query.filter( + sqlalchemy.and_( + models.ProjectUsers.user_id == user.user_id, + models.ProjectUsers.project_id == project.id, + ) + ).one_or_none() + assert project_user_after_addition + + +def test_add_existing_user_to_existing_project_no_mail_flag(client): + "Test that an e-mail notification is not send when the --no-mail flag is used" + + user_copy = existing_research_user_to_existing_project.copy() + project_id = user_copy.pop("project") + new_status = {"new_status": "Available"} + user_copy["send_email"] = False + token = tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client) + + # make project available prior to test, otherwise an e-mail is never sent. + response = client.post( + tests.DDSEndpoint.PROJECT_STATUS, + headers=token, + query_string={"project": project_id}, + data=json.dumps(new_status), + content_type="application/json", + ) + + # Test mail sending is suppressed + + with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: + with unittest.mock.patch.object( + dds_web.api.user.AddUser, "compose_and_send_email_to_user" + ) as mock_mail_func: + print(user_copy) + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=token, + query_string={"project": project_id}, + data=json.dumps(user_copy), + content_type="application/json", + ) + # assert that no mail is being sent. + assert mock_mail_func.called == False + assert mock_mail_send.call_count == 0 + + assert response.status_code == http.HTTPStatus.OK + assert "An e-mail notification has not been sent." in response.json["message"] + + +def test_add_existing_user_to_existing_project_after_release(client): + user_copy = existing_research_user_to_existing_project.copy() + project_id = user_copy.pop("project") + + project = models.Project.query.filter_by(public_id=project_id).one_or_none() + user = models.Email.query.filter_by( + email=existing_research_user_to_existing_project["email"] + ).one_or_none() + project_user_before_addition = models.ProjectUsers.query.filter( + sqlalchemy.and_( + models.ProjectUsers.user_id == user.user_id, + models.ProjectUsers.project_id == project.id, + ) + ).one_or_none() + assert project_user_before_addition is None + + # release project + response = client.post( + tests.DDSEndpoint.PROJECT_STATUS, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + query_string={"project": project_id}, + json={"new_status": "Available"}, + ) + assert response.status_code == http.HTTPStatus.OK + assert project.current_status == "Available" + + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + query_string={"project": project_id}, + json=user_copy, + ) + assert response.status_code == http.HTTPStatus.OK + + project_user_after_addition = models.ProjectUsers.query.filter( + sqlalchemy.and_( + models.ProjectUsers.user_id == user.user_id, + models.ProjectUsers.project_id == project.id, + ) + ).one_or_none() + assert project_user_after_addition + +def test_add_existing_user_to_nonexistent_proj(client): + user_copy = existing_research_user_to_nonexistent_proj.copy() + project = user_copy.pop("project") + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + query_string={"project": project}, + json=user_copy, + ) + assert response.status_code == http.HTTPStatus.BAD_REQUEST + + +def test_existing_user_change_ownership(client): + project = models.Project.query.filter_by( + public_id=change_owner_existing_user["project"] + ).one_or_none() + user = models.Email.query.filter_by(email=change_owner_existing_user["email"]).one_or_none() + project_user = models.ProjectUsers.query.filter( + sqlalchemy.and_( + models.ProjectUsers.user_id == user.user_id, + models.ProjectUsers.project_id == project.id, + ) + ).one_or_none() + + assert not project_user.owner + + user_new_owner_status = change_owner_existing_user.copy() + project = user_new_owner_status.pop("project") + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + query_string={"project": project}, + json=user_new_owner_status, + ) + + assert response.status_code == http.HTTPStatus.OK + + db.session.refresh(project_user) + + assert project_user.owner + + +def test_existing_user_change_ownership_same_permissions(client): + user_same_ownership = submit_with_same_ownership.copy() + project = user_same_ownership.pop("project") + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + query_string={"project": project}, + json=user_same_ownership, + ) + assert response.status_code == http.HTTPStatus.FORBIDDEN + + +def test_add_existing_user_with_unsuitable_role(client): + user_with_unsuitable_role = existing_research_user_to_existing_project.copy() + user_with_unsuitable_role["role"] = "Unit Admin" + project = user_with_unsuitable_role.pop("project") + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + query_string={"project": project}, + json=user_with_unsuitable_role, + ) + assert response.status_code == http.HTTPStatus.FORBIDDEN + + +# Invite to project ########################################################### Invite to project # + + +def test_invite_with_project_by_unituser(client): + "Test that a new invite including a project can be created" + + project = existing_project + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + query_string={"project": project}, + json=first_new_user, + ) + assert response.status_code == http.HTTPStatus.OK + + invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() + assert invited_user + assert invited_user.email == first_new_user["email"] + assert invited_user.role == first_new_user["role"] + + assert invited_user.nonce is not None + assert invited_user.public_key is not None + assert invited_user.private_key is not None + + project_invite_keys = invited_user.project_invite_keys + assert len(project_invite_keys) == 1 + assert project_invite_keys[0].project.public_id == project + assert not project_invite_keys[0].owner + + +def test_add_project_to_existing_invite_by_unituser(client): + "Test that a project can be associated with an existing invite" + + # Create invite upfront + + project = existing_project + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + json=first_new_user, + ) + assert response.status_code == http.HTTPStatus.OK + + invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() + + # Check that the invite has no projects yet + + assert invited_user + assert len(invited_user.project_invite_keys) == 0 + + # Add project to existing invite + + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + query_string={"project": project}, + json=first_new_user, + ) + + assert response.status_code == http.HTTPStatus.OK + + # Check that the invite has now a project association + project_invite_keys = invited_user.project_invite_keys + assert len(project_invite_keys) == 1 + assert project_invite_keys[0].project.public_id == project + assert not project_invite_keys[0].owner + + +def test_update_project_to_existing_invite_by_unituser(client): + "Test that project ownership can be updated for an existing invite" + + # Create Invite upfront + + project = existing_project + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + query_string={"project": project}, + json=first_new_user, + ) + assert response.status_code == http.HTTPStatus.OK + + project_obj = models.Project.query.filter_by(public_id=existing_project).one_or_none() + invite_obj = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() + + project_invite = models.ProjectInviteKeys.query.filter( + sqlalchemy.and_( + models.ProjectInviteKeys.invite_id == invite_obj.id, + models.ProjectUserKeys.project_id == project_obj.id, + ) + ).one_or_none() -# def test_update_project_to_existing_invite_by_unituser(client): -# "Test that project ownership can be updated for an existing invite" + assert project_invite + assert not project_invite.owner -# # Create Invite upfront - -# project = existing_project -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# query_string={"project": project}, -# json=first_new_user, -# ) -# assert response.status_code == http.HTTPStatus.OK - -# project_obj = models.Project.query.filter_by(public_id=existing_project).one_or_none() -# invite_obj = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + query_string={"project": project}, + json=first_new_owner, + ) + + assert response.status_code == http.HTTPStatus.OK + + db.session.refresh(project_invite) + + assert project_invite.owner + + +def test_invited_as_owner_and_researcher_to_different_project(client): + "Test that an invite can be owner of one project and researcher of another" + + # Create Invite upfront as owner + + project = existing_project + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + query_string={"project": project}, + json=first_new_owner, + ) + assert response.status_code == http.HTTPStatus.OK + + # Perform second invite as researcher + project2 = existing_project_2 + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), + query_string={"project": project2}, + json=first_new_user, + ) + assert response.status_code == http.HTTPStatus.OK + + project_obj_owner = models.Project.query.filter_by(public_id=existing_project).one_or_none() + project_obj_not_owner = models.Project.query.filter_by( + public_id=existing_project_2 + ).one_or_none() + + invite_obj = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() + + project_invite_owner = models.ProjectInviteKeys.query.filter( + sqlalchemy.and_( + models.ProjectInviteKeys.invite_id == invite_obj.id, + models.ProjectInviteKeys.project_id == project_obj_owner.id, + ) + ).one_or_none() + + assert project_invite_owner + assert project_invite_owner.owner + + project_invite_not_owner = models.ProjectInviteKeys.query.filter( + sqlalchemy.and_( + models.ProjectInviteKeys.invite_id == invite_obj.id, + models.ProjectInviteKeys.project_id == project_obj_not_owner.id, + ) + ).one_or_none() + + assert project_invite_not_owner + assert not project_invite_not_owner.owner + + # Owner or not should not be stored on the invite + assert invite_obj.role == "Researcher" + + +def test_invite_to_project_by_project_owner(client): + "Test that a project owner can invite to its project" + + project = existing_project + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["projectowner"]).token(client), + query_string={"project": project}, + json=first_new_user, + ) + assert response.status_code == http.HTTPStatus.OK + + invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() + assert invited_user + assert invited_user.email == first_new_user["email"] + assert invited_user.role == first_new_user["role"] + + assert invited_user.nonce is not None + assert invited_user.public_key is not None + assert invited_user.private_key is not None + + project_invite_keys = invited_user.project_invite_keys + assert len(project_invite_keys) == 1 + assert project_invite_keys[0].project.public_id == project + assert not project_invite_keys[0].owner + + +def test_add_anyuser_to_project_with_superadmin(client): + """Super admins cannot invite to project.""" + project = existing_project + for x in [first_new_user, first_new_owner, new_unit_user, new_unit_admin]: + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client), + query_string={"project": project}, + json=x, + ) + assert response.status_code == http.HTTPStatus.BAD_REQUEST + + # An invite should not have been created + invited_user = models.Invite.query.filter_by(email=x["email"]).one_or_none() + assert not invited_user -# project_invite = models.ProjectInviteKeys.query.filter( -# sqlalchemy.and_( -# models.ProjectInviteKeys.invite_id == invite_obj.id, -# models.ProjectUserKeys.project_id == project_obj.id, -# ) -# ).one_or_none() -# assert project_invite -# assert not project_invite.owner +def test_add_unituser_and_admin_no_unit_with_superadmin(client): + """A super admin needs to specify a unit to be able to invite unit users.""" + project = existing_project + for x in [new_unit_user, new_unit_admin]: + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client), + json=x, + ) -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# query_string={"project": project}, -# json=first_new_owner, -# ) - -# assert response.status_code == http.HTTPStatus.OK - -# db.session.refresh(project_invite) - -# assert project_invite.owner - - -# def test_invited_as_owner_and_researcher_to_different_project(client): -# "Test that an invite can be owner of one project and researcher of another" - -# # Create Invite upfront as owner - -# project = existing_project -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# query_string={"project": project}, -# json=first_new_owner, -# ) -# assert response.status_code == http.HTTPStatus.OK - -# # Perform second invite as researcher -# project2 = existing_project_2 -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), -# query_string={"project": project2}, -# json=first_new_user, -# ) -# assert response.status_code == http.HTTPStatus.OK - -# project_obj_owner = models.Project.query.filter_by(public_id=existing_project).one_or_none() -# project_obj_not_owner = models.Project.query.filter_by( -# public_id=existing_project_2 -# ).one_or_none() - -# invite_obj = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() - -# project_invite_owner = models.ProjectInviteKeys.query.filter( -# sqlalchemy.and_( -# models.ProjectInviteKeys.invite_id == invite_obj.id, -# models.ProjectInviteKeys.project_id == project_obj_owner.id, -# ) -# ).one_or_none() - -# assert project_invite_owner -# assert project_invite_owner.owner - -# project_invite_not_owner = models.ProjectInviteKeys.query.filter( -# sqlalchemy.and_( -# models.ProjectInviteKeys.invite_id == invite_obj.id, -# models.ProjectInviteKeys.project_id == project_obj_not_owner.id, -# ) -# ).one_or_none() - -# assert project_invite_not_owner -# assert not project_invite_not_owner.owner - -# # Owner or not should not be stored on the invite -# assert invite_obj.role == "Researcher" - - -# def test_invite_to_project_by_project_owner(client): -# "Test that a project owner can invite to its project" - -# project = existing_project -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["projectowner"]).token(client), -# query_string={"project": project}, -# json=first_new_user, -# ) -# assert response.status_code == http.HTTPStatus.OK - -# invited_user = models.Invite.query.filter_by(email=first_new_user["email"]).one_or_none() -# assert invited_user -# assert invited_user.email == first_new_user["email"] -# assert invited_user.role == first_new_user["role"] - -# assert invited_user.nonce is not None -# assert invited_user.public_key is not None -# assert invited_user.private_key is not None - -# project_invite_keys = invited_user.project_invite_keys -# assert len(project_invite_keys) == 1 -# assert project_invite_keys[0].project.public_id == project -# assert not project_invite_keys[0].owner - - -# def test_add_anyuser_to_project_with_superadmin(client): -# """Super admins cannot invite to project.""" -# project = existing_project -# for x in [first_new_user, first_new_owner, new_unit_user, new_unit_admin]: -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client), -# query_string={"project": project}, -# json=x, -# ) -# assert response.status_code == http.HTTPStatus.BAD_REQUEST - -# # An invite should not have been created -# invited_user = models.Invite.query.filter_by(email=x["email"]).one_or_none() -# assert not invited_user - - -# def test_add_unituser_and_admin_no_unit_with_superadmin(client): -# """A super admin needs to specify a unit to be able to invite unit users.""" -# project = existing_project -# for x in [new_unit_user, new_unit_admin]: -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client), -# json=x, -# ) - -# assert "You need to specify a unit" in response.json["message"] -# assert response.status_code == http.HTTPStatus.BAD_REQUEST - -# invited_user = models.Invite.query.filter_by(email=x["email"]).one_or_none() -# assert not invited_user - - -# def test_add_researchuser_project_no_access_unit_admin_and_personnel(client): -# """A unit admin and personnel should not be able to give access to a project -# which they themselves have lost access to.""" -# # Make sure the project exists -# project = models.Project.query.filter_by(public_id=existing_project).one_or_none() -# assert project - -# for inviter in ["unitadmin", "unituser", "projectowner"]: -# # Check that the unit admin has access to the project first -# project_user_key = models.ProjectUserKeys.query.filter_by( -# user_id=inviter, project_id=project.id -# ).one_or_none() -# assert project_user_key - -# # Remove the project access (for test) -# db.session.delete(project_user_key) - -# # Make sure the project access does not exist now -# project_user_key = models.ProjectUserKeys.query.filter_by( -# user_id=inviter, project_id=project.id -# ).one_or_none() -# assert not project_user_key - -# for x in [first_new_user, first_new_owner]: -# # Make sure there is no ongoing invite -# invited_user_before = models.Invite.query.filter_by(email=x["email"]).one_or_none() -# if invited_user_before: -# db.session.delete(invited_user_before) -# invited_user_before = models.Invite.query.filter_by(email=x["email"]).one_or_none() -# assert not invited_user_before - -# # Attempt invite -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS[inviter]).token(client), -# json=x, -# query_string={"project": project.public_id}, -# ) - -# # The invite should still be done, but they can't invite to a project -# invited_user = models.Invite.query.filter_by(email=x["email"]).one_or_none() -# assert invited_user - -# # Make sure there are no project v -# assert not invited_user.project_invite_keys - -# # There should be an error message -# assert response.status_code == http.HTTPStatus.BAD_REQUEST -# assert "The user could not be added to the project(s)" in response.json["message"] - -# # Verify ok error messages -# assert "errors" in response.json -# assert project.public_id in response.json["errors"] -# assert ( -# "You do not have access to the specified project." -# in response.json["errors"][project.public_id] -# ) - - -# # Invite without email -# def test_invite_without_email(client): -# """The email is required.""" -# user_no_email = first_new_user.copy() -# user_no_email.pop("email") - -# for inviter in ["superadmin", "unitadmin", "unituser"]: -# # Attempt invite -# response = client.post( -# tests.DDSEndpoint.USER_ADD, -# headers=tests.UserAuth(tests.USER_CREDENTIALS[inviter]).token(client), -# json=user_no_email, -# # query_string={"project": existing_project}, -# ) - -# # There should be an error message -# assert response.status_code == http.HTTPStatus.BAD_REQUEST -# assert "Email address required to add or invite." in response.json["message"] + assert "You need to specify a unit" in response.json["message"] + assert response.status_code == http.HTTPStatus.BAD_REQUEST + + invited_user = models.Invite.query.filter_by(email=x["email"]).one_or_none() + assert not invited_user + + +def test_add_researchuser_project_no_access_unit_admin_and_personnel(client): + """A unit admin and personnel should not be able to give access to a project + which they themselves have lost access to.""" + # Make sure the project exists + project = models.Project.query.filter_by(public_id=existing_project).one_or_none() + assert project + + for inviter in ["unitadmin", "unituser", "projectowner"]: + # Check that the unit admin has access to the project first + project_user_key = models.ProjectUserKeys.query.filter_by( + user_id=inviter, project_id=project.id + ).one_or_none() + assert project_user_key + + # Remove the project access (for test) + db.session.delete(project_user_key) + + # Make sure the project access does not exist now + project_user_key = models.ProjectUserKeys.query.filter_by( + user_id=inviter, project_id=project.id + ).one_or_none() + assert not project_user_key + + for x in [first_new_user, first_new_owner]: + # Make sure there is no ongoing invite + invited_user_before = models.Invite.query.filter_by(email=x["email"]).one_or_none() + if invited_user_before: + db.session.delete(invited_user_before) + invited_user_before = models.Invite.query.filter_by(email=x["email"]).one_or_none() + assert not invited_user_before + + # Attempt invite + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS[inviter]).token(client), + json=x, + query_string={"project": project.public_id}, + ) + + # The invite should still be done, but they can't invite to a project + invited_user = models.Invite.query.filter_by(email=x["email"]).one_or_none() + assert invited_user + + # Make sure there are no project v + assert not invited_user.project_invite_keys + + # There should be an error message + assert response.status_code == http.HTTPStatus.BAD_REQUEST + assert "The user could not be added to the project(s)" in response.json["message"] + + # Verify ok error messages + assert "errors" in response.json + assert project.public_id in response.json["errors"] + assert ( + "You do not have access to the specified project." + in response.json["errors"][project.public_id] + ) + + +# Invite without email +def test_invite_without_email(client): + """The email is required.""" + user_no_email = first_new_user.copy() + user_no_email.pop("email") + + for inviter in ["superadmin", "unitadmin", "unituser"]: + # Attempt invite + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS[inviter]).token(client), + json=user_no_email, + # query_string={"project": existing_project}, + ) + + # There should be an error message + assert response.status_code == http.HTTPStatus.BAD_REQUEST + assert "Email address required to add or invite." in response.json["message"] # Invite super admin with unit admin From a73462c4b27275fa83f2ec345b5ad5f409e248d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Sun, 13 Mar 2022 14:09:00 +0100 Subject: [PATCH 053/293] Update dds_web/api/dds_decorators.py --- dds_web/api/dds_decorators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/api/dds_decorators.py b/dds_web/api/dds_decorators.py index 8cfd0ebe5..fbd78ae0b 100644 --- a/dds_web/api/dds_decorators.py +++ b/dds_web/api/dds_decorators.py @@ -123,7 +123,7 @@ def init_resource(self, *args, **kwargs): except sqlalchemy.exc.SQLAlchemyError as sqlerr: raise DatabaseError(message=str(sqlerr)) from sqlerr except botocore.client.ClientError as clierr: - raise S3ConnectionError(message=str(clierr)) from sqlerr + raise S3ConnectionError(message=str(clierr)) from clierr return func(self, *args, **kwargs) From 652950092a5312a3949a8edc6ed481497140b760 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Sun, 13 Mar 2022 16:12:41 +0100 Subject: [PATCH 054/293] return if has access --- dds_web/api/project.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index f30032ed5..0841f5903 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -326,6 +326,12 @@ def get(self): # return ByteHours project_info.update({"Usage": proj_bhours, "Cost": proj_cost}) + project_info["Access"] = ( + models.ProjectUserKeys.query.filter_by( + project_id=p.id, user_id=current_user.username + ).count() + > 0 + ) all_projects.append(project_info) return_info = { From afd6a60d2df5bfe96881b43e58e0fe510a8c1497 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Sun, 13 Mar 2022 16:31:25 +0100 Subject: [PATCH 055/293] changelog row --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9548968c..cc9202d53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,3 +46,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - Introduce a separate error message if someone tried to add an unit user to projects individually. ([#1039](https://github.com/ScilifelabDataCentre/dds_web/pull/1039)) - Display an error message when the user makes too many authentication requests. ([#1034](https://github.com/ScilifelabDataCentre/dds_web/pull/1034)) +- When listing the projects, return whether or not the user has a project key for that particular project () \ No newline at end of file From 99f73190ae4bcf9e63ffe7e340376640e9a386df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Sun, 13 Mar 2022 16:32:28 +0100 Subject: [PATCH 056/293] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc9202d53..accc8b8c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,4 +46,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - Introduce a separate error message if someone tried to add an unit user to projects individually. ([#1039](https://github.com/ScilifelabDataCentre/dds_web/pull/1039)) - Display an error message when the user makes too many authentication requests. ([#1034](https://github.com/ScilifelabDataCentre/dds_web/pull/1034)) -- When listing the projects, return whether or not the user has a project key for that particular project () \ No newline at end of file +- When listing the projects, return whether or not the user has a project key for that particular project ([#1049](https://github.com/ScilifelabDataCentre/dds_web/pull/1049)) \ No newline at end of file From baa2a037ed3bfe3bbadf7246debbca48d23c2904 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Sun, 13 Mar 2022 16:33:47 +0100 Subject: [PATCH 057/293] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index accc8b8c8..e1db2995e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,4 +46,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - Introduce a separate error message if someone tried to add an unit user to projects individually. ([#1039](https://github.com/ScilifelabDataCentre/dds_web/pull/1039)) - Display an error message when the user makes too many authentication requests. ([#1034](https://github.com/ScilifelabDataCentre/dds_web/pull/1034)) -- When listing the projects, return whether or not the user has a project key for that particular project ([#1049](https://github.com/ScilifelabDataCentre/dds_web/pull/1049)) \ No newline at end of file +- When listing the projects, return whether or not the user has a project key for that particular project ([#1049](https://github.com/ScilifelabDataCentre/dds_web/pull/1049)) From ff1ce1cc7baf63609dd23955fe4e75c9a7262516 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Sun, 13 Mar 2022 23:48:55 +0100 Subject: [PATCH 058/293] new unitusers endpoint --- dds_web/api/__init__.py | 2 +- dds_web/api/user.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/dds_web/api/__init__.py b/dds_web/api/__init__.py index 606ed67ad..987cf1b44 100644 --- a/dds_web/api/__init__.py +++ b/dds_web/api/__init__.py @@ -67,6 +67,6 @@ def output_json(data, code, headers=None): api.add_resource(user.DeleteUserSelf, "/user/delete_self", endpoint="delete_user_self") api.add_resource(user.RemoveUserAssociation, "/user/access/revoke", endpoint="revoke_from_project") api.add_resource(user.UserActivation, "/user/activation", endpoint="user_activation") - +api.add_resource(user.UnitUsers, "/unit/users", endpoint="unit_users") # Invoicing ############################################################################ Invoicing # api.add_resource(user.ShowUsage, "/usage", endpoint="usage") diff --git a/dds_web/api/user.py b/dds_web/api/user.py index 64ed21a15..028fca884 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -884,3 +884,35 @@ def get(self): }, "project_usage": usage, } + + +class UnitUsers(flask_restful.Resource): + """List unit users.""" + + @auth.login_required(role=["Unit Admin", "Unit Personnel"]) + @logging_bind_request + def get(self): + """Get and return unit users within the unit the current user is connected to.""" + unit_users = {} + + if not auth.current_user().is_active: + raise ddserr.AccessDeniedError( + message=( + "Your account has been deactivated. " + "You cannot list the users within your unit." + ) + ) + + keys = ["Name", "Username", "Email", "Role", "Active"] + unit_users = [ + { + "Name": user.name, + "Username": user.username, + "Email": user.primary_email, + "Role": user.role, + "Active": user.is_active, + } + for user in auth.current_user().unit.users + ] + + return {"users": unit_users, "keys": keys, "unit": auth.current_user().unit.name} From d6feb0f552750e50c9313686412038650277a32e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Sun, 13 Mar 2022 23:52:52 +0100 Subject: [PATCH 059/293] Revert "new unitusers endpoint" This reverts commit ff1ce1cc7baf63609dd23955fe4e75c9a7262516. --- dds_web/api/__init__.py | 2 +- dds_web/api/user.py | 32 -------------------------------- 2 files changed, 1 insertion(+), 33 deletions(-) diff --git a/dds_web/api/__init__.py b/dds_web/api/__init__.py index 987cf1b44..606ed67ad 100644 --- a/dds_web/api/__init__.py +++ b/dds_web/api/__init__.py @@ -67,6 +67,6 @@ def output_json(data, code, headers=None): api.add_resource(user.DeleteUserSelf, "/user/delete_self", endpoint="delete_user_self") api.add_resource(user.RemoveUserAssociation, "/user/access/revoke", endpoint="revoke_from_project") api.add_resource(user.UserActivation, "/user/activation", endpoint="user_activation") -api.add_resource(user.UnitUsers, "/unit/users", endpoint="unit_users") + # Invoicing ############################################################################ Invoicing # api.add_resource(user.ShowUsage, "/usage", endpoint="usage") diff --git a/dds_web/api/user.py b/dds_web/api/user.py index 028fca884..64ed21a15 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -884,35 +884,3 @@ def get(self): }, "project_usage": usage, } - - -class UnitUsers(flask_restful.Resource): - """List unit users.""" - - @auth.login_required(role=["Unit Admin", "Unit Personnel"]) - @logging_bind_request - def get(self): - """Get and return unit users within the unit the current user is connected to.""" - unit_users = {} - - if not auth.current_user().is_active: - raise ddserr.AccessDeniedError( - message=( - "Your account has been deactivated. " - "You cannot list the users within your unit." - ) - ) - - keys = ["Name", "Username", "Email", "Role", "Active"] - unit_users = [ - { - "Name": user.name, - "Username": user.username, - "Email": user.primary_email, - "Role": user.role, - "Active": user.is_active, - } - for user in auth.current_user().unit.users - ] - - return {"users": unit_users, "keys": keys, "unit": auth.current_user().unit.name} From 48b291a3422ac763c1aefd8a929b456c97a0cb12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Sun, 13 Mar 2022 23:55:44 +0100 Subject: [PATCH 060/293] add new endpoint for listing unit users --- dds_web/api/__init__.py | 1 + dds_web/api/user.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/dds_web/api/__init__.py b/dds_web/api/__init__.py index 606ed67ad..0d84495f0 100644 --- a/dds_web/api/__init__.py +++ b/dds_web/api/__init__.py @@ -67,6 +67,7 @@ def output_json(data, code, headers=None): api.add_resource(user.DeleteUserSelf, "/user/delete_self", endpoint="delete_user_self") api.add_resource(user.RemoveUserAssociation, "/user/access/revoke", endpoint="revoke_from_project") api.add_resource(user.UserActivation, "/user/activation", endpoint="user_activation") +api.add_resource(user.UnitUsers, "/unit/users", endpoint="unit_users") # Invoicing ############################################################################ Invoicing # api.add_resource(user.ShowUsage, "/usage", endpoint="usage") diff --git a/dds_web/api/user.py b/dds_web/api/user.py index 64ed21a15..028fca884 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -884,3 +884,35 @@ def get(self): }, "project_usage": usage, } + + +class UnitUsers(flask_restful.Resource): + """List unit users.""" + + @auth.login_required(role=["Unit Admin", "Unit Personnel"]) + @logging_bind_request + def get(self): + """Get and return unit users within the unit the current user is connected to.""" + unit_users = {} + + if not auth.current_user().is_active: + raise ddserr.AccessDeniedError( + message=( + "Your account has been deactivated. " + "You cannot list the users within your unit." + ) + ) + + keys = ["Name", "Username", "Email", "Role", "Active"] + unit_users = [ + { + "Name": user.name, + "Username": user.username, + "Email": user.primary_email, + "Role": user.role, + "Active": user.is_active, + } + for user in auth.current_user().unit.users + ] + + return {"users": unit_users, "keys": keys, "unit": auth.current_user().unit.name} From d9617f6e0f71ad4763bc6368002e908945ac792e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Mon, 14 Mar 2022 08:35:02 +0100 Subject: [PATCH 061/293] added tests for unit users --- tests/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/__init__.py b/tests/__init__.py index 4be019863..37e260d1c 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -148,6 +148,9 @@ class DDSEndpoint: USER_DELETE_SELF = BASE_ENDPOINT + "/user/delete_self" USER_CONFIRM_DELETE = "/confirm_deletion/" + # List users + LIST_UNIT_USERS = BASE_ENDPOINT + "/unit/users" + # Authentication - user and project ENCRYPTED_TOKEN = BASE_ENDPOINT + "/user/encrypted_token" SECOND_FACTOR = BASE_ENDPOINT + "/user/second_factor" From c76d619110526f4fd9d2d0b1884200bdf2a99c8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Mon, 14 Mar 2022 08:36:53 +0100 Subject: [PATCH 062/293] add tests --- tests/test_users_list.py | 92 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 tests/test_users_list.py diff --git a/tests/test_users_list.py b/tests/test_users_list.py new file mode 100644 index 000000000..6cf7ffe87 --- /dev/null +++ b/tests/test_users_list.py @@ -0,0 +1,92 @@ +# IMPORTS ################################################################################ IMPORTS # + +# Standard library +import http +import json +from urllib import response + +# Own +from dds_web import db +from dds_web.api import user +from dds_web.database import models +import tests + +# CONFIG ################################################################################## CONFIG # + +users = { + "Researcher": "researchuser", + "Unit Personnel": "unituser", + "Unit Admin": "unitadmin", + "Super Admin": "superadmin", +} + +# TESTS #################################################################################### TESTS # + + +def get_token(username, client): + return tests.UserAuth(tests.USER_CREDENTIALS[username]).token(client) + + +def test_list_unitusers_with_researcher(client): + """Researchers cannot list unit users.""" + token = get_token(username=users["Researcher"], client=client) + response = client.get(tests.DDSEndpoint.LIST_UNIT_USERS, headers=token) + assert response.status_code == http.HTTPStatus.FORBIDDEN + + +def test_list_unitusers_with_super_admin(client): + """Super admins will be able to list unit users, but not right now.""" + token = get_token(username=users["Super Admin"], client=client) + response = client.get(tests.DDSEndpoint.LIST_UNIT_USERS, headers=token) + assert response.status_code == http.HTTPStatus.FORBIDDEN + + +def test_list_unitusers_with_unit_personnel_and_admin_deactivated(client): + """Unit Personnel should be able to list the users within a unit.""" + # Deactivate user + for u in ["Unit Personnel", "Unit Admin"]: + # Get token + token = get_token(username=users[u], client=client) + + user = models.User.query.get(users[u]) + user.active = False + db.session.commit() + + # Try to list users - should only work if active - not now + response = client.get(tests.DDSEndpoint.LIST_UNIT_USERS, headers=token) + + # Unauth and not forbidden because the user object is not returned from the token + assert response.status_code == http.HTTPStatus.UNAUTHORIZED + + +def test_list_unitusers_with_unit_personnel_and_admin_ok(client): + # Active unit users should be able to list unit users + for u in ["Unit Personnel", "Unit Admin"]: + # Get token + token = get_token(username=users[u], client=client) + + # Get users + response = client.get(tests.DDSEndpoint.LIST_UNIT_USERS, headers=token) + assert response.status_code == http.HTTPStatus.OK + + keys_in_response = response.json["keys"] + unit_in_response = response.json["unit"] + users_in_response = response.json["users"] + + assert keys_in_response + + user_object = models.User.query.get(users[u]) + assert user_object.unit.name == unit_in_response + + all_users = user_object.unit.users + + # ["Name", "Username", "Email", "Role", "Active"] + for dbrow in user_object.unit.users: + expected = { + "Name": dbrow.name, + "Username": dbrow.username, + "Email": dbrow.primary_email, + "Role": dbrow.role, + "Active": dbrow.is_active, + } + assert expected in users_in_response From a1d2f2165dceb5e792babbf799cd8c8c7d971eda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Mon, 14 Mar 2022 08:39:55 +0100 Subject: [PATCH 063/293] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9548968c..4c84bf619 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,3 +46,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - Introduce a separate error message if someone tried to add an unit user to projects individually. ([#1039](https://github.com/ScilifelabDataCentre/dds_web/pull/1039)) - Display an error message when the user makes too many authentication requests. ([#1034](https://github.com/ScilifelabDataCentre/dds_web/pull/1034)) +- New endpoint for Unit Personnel and Admins to list the other Unit Personnel / Admins within their project From 767b38293ed02323ccf077bade6643306d61e9de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Mon, 14 Mar 2022 08:41:50 +0100 Subject: [PATCH 064/293] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c84bf619..10e568c86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,4 +46,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - Introduce a separate error message if someone tried to add an unit user to projects individually. ([#1039](https://github.com/ScilifelabDataCentre/dds_web/pull/1039)) - Display an error message when the user makes too many authentication requests. ([#1034](https://github.com/ScilifelabDataCentre/dds_web/pull/1034)) -- New endpoint for Unit Personnel and Admins to list the other Unit Personnel / Admins within their project +- New endpoint for Unit Personnel and Admins to list the other Unit Personnel / Admins within their project ([#1050](https://github.com/ScilifelabDataCentre/dds_web/pull/1050)) From 7f235ea2c8f59e22f987fd8d2023ae18c91b9250 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Mon, 14 Mar 2022 09:49:49 +0100 Subject: [PATCH 065/293] Update dds_web/api/user.py --- dds_web/api/user.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index 0a63d7bdf..c0675745e 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -6,7 +6,6 @@ # Standard library import os -from pyexpat import model from re import T import smtplib import time From c29581790a6b4be8c1eacdbf2655f1a9e94966db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Mon, 14 Mar 2022 09:50:04 +0100 Subject: [PATCH 066/293] Update dds_web/api/user.py --- dds_web/api/user.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index c0675745e..e52f97545 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -6,7 +6,6 @@ # Standard library import os -from re import T import smtplib import time import datetime From ba731fdd926226f6d1651ed76c83504374dc7c8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Mon, 14 Mar 2022 09:50:18 +0100 Subject: [PATCH 067/293] Update dds_web/api/user.py --- dds_web/api/user.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index e52f97545..d3887a1ea 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -9,7 +9,6 @@ import smtplib import time import datetime -from tkinter.tix import Tree # Installed import flask From 66dd210fbacf469c0c0911c8eec7638155b7a96d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Mon, 14 Mar 2022 10:42:06 +0100 Subject: [PATCH 068/293] Update tests/test_users_list.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Linus Östberg --- tests/test_users_list.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_users_list.py b/tests/test_users_list.py index 6cf7ffe87..588a4f334 100644 --- a/tests/test_users_list.py +++ b/tests/test_users_list.py @@ -2,8 +2,6 @@ # Standard library import http -import json -from urllib import response # Own from dds_web import db From 1e1ad9f805ae94b182413b63e05f9f333c0f8a4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Mon, 14 Mar 2022 13:19:52 +0100 Subject: [PATCH 069/293] make hotp invalid after password reset --- dds_web/database/models.py | 5 +++++ dds_web/web/user.py | 3 +++ 2 files changed, 8 insertions(+) diff --git a/dds_web/database/models.py b/dds_web/database/models.py index aacdb0849..a3430f2e6 100644 --- a/dds_web/database/models.py +++ b/dds_web/database/models.py @@ -461,6 +461,11 @@ def generate_HOTP_token(self): hotp = twofactor_hotp.HOTP(self.hotp_secret, 8, hashes.SHA512()) return hotp.generate(self.hotp_counter) + def reset_current_HOTP(self): + """Make the previous HOTP as invalid by nulling issue time and increasing counter.""" + self.hotp_issue_time = None + self.hotp_counter += 1 + def verify_HOTP(self, token): """Verify the HOTP token. diff --git a/dds_web/web/user.py b/dds_web/web/user.py index 5252859a8..e8a003506 100644 --- a/dds_web/web/user.py +++ b/dds_web/web/user.py @@ -372,6 +372,9 @@ def reset_password(token): # Validate form if form.validate_on_submit(): + # Clear out hotp + user.reset_current_HOTP() + # Delete project user keys for user for project_user_key in user.project_user_keys: db.session.delete(project_user_key) From f6fc208aee7577427a6a2e0a9efe722718220e87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Mon, 14 Mar 2022 13:21:46 +0100 Subject: [PATCH 070/293] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d7d9f2cb..0da3851ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,3 +48,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - Catch KeyNotFoundError when user tries to give access to a project they themselves do not have access to ([#1045](https://github.com/ScilifelabDataCentre/dds_web/pull/1045)) - Display an error message when the user makes too many authentication requests. ([#1034](https://github.com/ScilifelabDataCentre/dds_web/pull/1034)) - New endpoint for Unit Personnel and Admins to list the other Unit Personnel / Admins within their project ([#1050](https://github.com/ScilifelabDataCentre/dds_web/pull/1050)) +- Make previous HOTP invalid at password reset () From bd44287d66fc426770353bf8257be0bb99060d89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Mon, 14 Mar 2022 13:22:32 +0100 Subject: [PATCH 071/293] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0da3851ac..613758cfc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,4 +48,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - Catch KeyNotFoundError when user tries to give access to a project they themselves do not have access to ([#1045](https://github.com/ScilifelabDataCentre/dds_web/pull/1045)) - Display an error message when the user makes too many authentication requests. ([#1034](https://github.com/ScilifelabDataCentre/dds_web/pull/1034)) - New endpoint for Unit Personnel and Admins to list the other Unit Personnel / Admins within their project ([#1050](https://github.com/ScilifelabDataCentre/dds_web/pull/1050)) -- Make previous HOTP invalid at password reset () +- Make previous HOTP invalid at password reset ([#1054](https://github.com/ScilifelabDataCentre/dds_web/pull/1054)) From 93c6f545c354024d5295f5dad015e7c4a246ec75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Mon, 14 Mar 2022 13:38:47 +0100 Subject: [PATCH 072/293] don't remove unit --- dds_web/api/project.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 1d94f3ae3..3df5edd1a 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -224,7 +224,6 @@ def delete_project_info(self, proj): proj.description = None proj.pi = None proj.public_key = None - proj.unit_id = None proj.created_by = None # Delete User associations for user in proj.researchusers: From 8dac9b5dff1a8357477ddb0a4f8b5c4a0bb387e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Mon, 14 Mar 2022 13:40:46 +0100 Subject: [PATCH 073/293] check eligibility for upload --- dds_web/api/s3.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dds_web/api/s3.py b/dds_web/api/s3.py index 5d7d3ce17..329234ef6 100644 --- a/dds_web/api/s3.py +++ b/dds_web/api/s3.py @@ -20,6 +20,7 @@ DatabaseError, ) from dds_web.api.schemas import project_schemas +from dds_web.api.files import check_eligibility_for_upload #################################################################################################### # ENDPOINTS ############################################################################ ENDPOINTS # @@ -37,6 +38,8 @@ def get(self): # Verify project ID and access project = project_schemas.ProjectRequiredSchema().load(flask.request.args) + check_eligibility_for_upload(status=project.current_status) + try: sfsp_proj, keys, url, bucketname = ApiS3Connector(project=project).get_s3_info() except sqlalchemy.exc.SQLAlchemyError as sqlerr: From 35cc8918de3e4de53a34b085eace503140f183ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Mon, 14 Mar 2022 13:48:43 +0100 Subject: [PATCH 074/293] test fix --- dds_web/api/s3.py | 2 -- tests/test_project_status.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/dds_web/api/s3.py b/dds_web/api/s3.py index 329234ef6..c75a65ea0 100644 --- a/dds_web/api/s3.py +++ b/dds_web/api/s3.py @@ -38,8 +38,6 @@ def get(self): # Verify project ID and access project = project_schemas.ProjectRequiredSchema().load(flask.request.args) - check_eligibility_for_upload(status=project.current_status) - try: sfsp_proj, keys, url, bucketname = ApiS3Connector(project=project).get_s3_info() except sqlalchemy.exc.SQLAlchemyError as sqlerr: diff --git a/tests/test_project_status.py b/tests/test_project_status.py index 8ed4a09de..0d24eb722 100644 --- a/tests/test_project_status.py +++ b/tests/test_project_status.py @@ -32,7 +32,7 @@ "description", "pi", "public_key", - "unit_id", + # "unit_id", "created_by", # "is_active", # "date_updated", From f609e87f66be4d0e9d0c0b88c40857311bcd812c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Mon, 14 Mar 2022 15:56:02 +0100 Subject: [PATCH 075/293] check if ok to upload --- dds_web/api/s3.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dds_web/api/s3.py b/dds_web/api/s3.py index c75a65ea0..329234ef6 100644 --- a/dds_web/api/s3.py +++ b/dds_web/api/s3.py @@ -38,6 +38,8 @@ def get(self): # Verify project ID and access project = project_schemas.ProjectRequiredSchema().load(flask.request.args) + check_eligibility_for_upload(status=project.current_status) + try: sfsp_proj, keys, url, bucketname = ApiS3Connector(project=project).get_s3_info() except sqlalchemy.exc.SQLAlchemyError as sqlerr: From c09dd83f5dce4c40042e93e63007edf7b0c17553 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Mon, 14 Mar 2022 17:28:44 +0100 Subject: [PATCH 076/293] add reset password table --- dds_web/database/models.py | 21 +++++++++++++++++++++ dds_web/web/user.py | 31 +++++++++++++++++++++++++++++++ docker-compose.yml | 2 +- 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/dds_web/database/models.py b/dds_web/database/models.py index a3430f2e6..7736d194c 100644 --- a/dds_web/database/models.py +++ b/dds_web/database/models.py @@ -6,6 +6,7 @@ # Standard library import datetime +from enum import unique import os # Installed @@ -385,6 +386,7 @@ class User(flask_login.UserMixin, db.Model): deletion_request = db.relationship( "DeletionRequest", back_populates="requester", cascade="all, delete" ) + password_reset = db.relationship("PasswordReset", back_populates="user", cascade="all, delete") __mapper_args__ = {"polymorphic_on": type} # No polymorphic identity --> no create only user @@ -763,6 +765,25 @@ def __repr__(self): return f"" +class PasswordReset(db.Model): + """Keep track of password resets.""" + + # Table setup + __tablename__ = "password_resets" + __table_args__ = {"extend_existing": True} + + # Primary Key + id = db.Column(db.Integer, primary_key=True, autoincrement=True) + + user_id = db.Column(db.String(50), db.ForeignKey("users.username", ondelete="CASCADE")) + user = db.relationship("User", back_populates="password_reset") + + email = db.Column(db.String(254), unique=True, nullable=False) + issued = db.Column(db.DateTime(), unique=False, nullable=False) + + valid = db.Column(db.Boolean, unique=False, nullable=False, default=True) + + class File(db.Model): """ Data model for files. diff --git a/dds_web/web/user.py b/dds_web/web/user.py index e8a003506..db913ceaf 100644 --- a/dds_web/web/user.py +++ b/dds_web/web/user.py @@ -336,6 +336,20 @@ def request_reset_password(): ), additional_claims={"rst": "pwd"}, ) + + # Create row in password reset table + ongoing_password_reset = models.PasswordReset.query.filter_by( + email=email.email + ).one_or_none() + if ongoing_password_reset: + ongoing_password_reset.issued = dds_web.utils.current_time() + else: + new_password_reset = models.PasswordReset( + user=email.user, email=email.email, issued=dds_web.utils.current_time() + ) + db.session.add(new_password_reset) + db.session.commit() + dds_web.utils.send_reset_email(email_row=email, token=token) flask.flash("An email has been sent with instructions to reset your password.") return flask.redirect(flask.url_for("auth_blueprint.login")) @@ -363,6 +377,20 @@ def reset_password(token): if not user.is_active: flask.flash("Your account is not active. You cannot reset your password.", "warning") return flask.redirect(flask.url_for("pages.home")) + + password_reset_row = models.PasswordReset.query.filter_by( + user_id=user.username + ).one_or_none() + if not password_reset_row: + flask.flash("No information on requested password reset.") + return flask.redirect(flask.url_for("pages.home")) + if not password_reset_row.valid: + flask.flash( + "You have already used this link to change your password. " + "Please request a new password reset if you wish to continue." + ) + return flask.redirect(flask.url_for("pages.home")) + except ddserr.AuthenticationError: flask.flash("That is an invalid or expired token", "warning") return flask.redirect(flask.url_for("pages.home")) @@ -388,6 +416,9 @@ def reset_password(token): # Update user password user.password = form.password.data + + # Set password reset row as invalid + password_reset_row.valid = False db.session.commit() flask.flash("Your password has been updated! You are now able to log in.", "success") diff --git a/docker-compose.yml b/docker-compose.yml index f59a1b0a8..ece3acba6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,7 +30,7 @@ services: # source: ./Dockerfiles/mariadb/db-encrypt # target: /etc/mysql/encrypt - # Development service to watch for changes to SCSS files and recompile static CSS + # Development service to watch for changes to SCSS files and recompile static CSS node_builder: container_name: dds_nodebuilder build: From 09a39628e848a5f5317a6dcb870912f6f77f2def Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Mon, 14 Mar 2022 17:29:12 +0100 Subject: [PATCH 077/293] migration --- .../1256117ad629_add_password_reset.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 migrations/versions/1256117ad629_add_password_reset.py diff --git a/migrations/versions/1256117ad629_add_password_reset.py b/migrations/versions/1256117ad629_add_password_reset.py new file mode 100644 index 000000000..47474f9fd --- /dev/null +++ b/migrations/versions/1256117ad629_add_password_reset.py @@ -0,0 +1,38 @@ +"""add_password_reset + +Revision ID: 1256117ad629 +Revises: 666003748d14 +Create Date: 2022-03-14 16:23:44.287254 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = "1256117ad629" +down_revision = "666003748d14" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "password_resets", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("user_id", sa.String(length=50), nullable=True), + sa.Column("email", sa.String(length=254), nullable=False), + sa.Column("issued", sa.DateTime(), nullable=False), + sa.Column("valid", sa.Boolean(), nullable=False), + sa.ForeignKeyConstraint(["user_id"], ["users.username"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("email"), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("password_resets") + # ### end Alembic commands ### From 43e4da735c8e861c9a585f47eddf33c14b6116ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Mon, 14 Mar 2022 17:31:00 +0100 Subject: [PATCH 078/293] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 613758cfc..6bd9eec84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,3 +49,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - Display an error message when the user makes too many authentication requests. ([#1034](https://github.com/ScilifelabDataCentre/dds_web/pull/1034)) - New endpoint for Unit Personnel and Admins to list the other Unit Personnel / Admins within their project ([#1050](https://github.com/ScilifelabDataCentre/dds_web/pull/1050)) - Make previous HOTP invalid at password reset ([#1054](https://github.com/ScilifelabDataCentre/dds_web/pull/1054)) +- New PasswordReset table to keep track of when a user has requested a password reset () From 423d1d8e54d9dabb8492f02732130eae9bde201f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Mon, 14 Mar 2022 17:55:55 +0100 Subject: [PATCH 079/293] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bd9eec84..063429180 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,4 +49,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - Display an error message when the user makes too many authentication requests. ([#1034](https://github.com/ScilifelabDataCentre/dds_web/pull/1034)) - New endpoint for Unit Personnel and Admins to list the other Unit Personnel / Admins within their project ([#1050](https://github.com/ScilifelabDataCentre/dds_web/pull/1050)) - Make previous HOTP invalid at password reset ([#1054](https://github.com/ScilifelabDataCentre/dds_web/pull/1054)) -- New PasswordReset table to keep track of when a user has requested a password reset () +- New PasswordReset table to keep track of when a user has requested a password reset ([#1058](https://github.com/ScilifelabDataCentre/dds_web/pull/1058)) From bb1fb228ff8704b5867db95111d8d9cd8a44b5e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Mon, 14 Mar 2022 17:56:17 +0100 Subject: [PATCH 080/293] Update dds_web/database/models.py --- dds_web/database/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dds_web/database/models.py b/dds_web/database/models.py index 7736d194c..8ce24741b 100644 --- a/dds_web/database/models.py +++ b/dds_web/database/models.py @@ -6,7 +6,6 @@ # Standard library import datetime -from enum import unique import os # Installed From 51562b35f49082e82850046218c6d0958c6e2b7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Mon, 14 Mar 2022 20:48:48 +0100 Subject: [PATCH 081/293] add password reset row to fix a test --- tests/test_user_reset_password.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/test_user_reset_password.py b/tests/test_user_reset_password.py index fb8b8afb2..cccacbb1e 100644 --- a/tests/test_user_reset_password.py +++ b/tests/test_user_reset_password.py @@ -1,6 +1,7 @@ from cryptography.hazmat.primitives.ciphers.aead import AESGCM import datetime import http +import dds_cli import flask import flask_mail import pytest @@ -12,6 +13,8 @@ from dds_web.security.project_user_keys import generate_invite_key_pair from dds_web.security.tokens import encrypted_jwt_token +researcher = {"username": "researchuser", "email": "researchuser@mailtrap.io"} + def test_request_reset_password_no_form(client): with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: @@ -115,11 +118,18 @@ def get_valid_reset_token(username, expires_in=3600): def test_reset_password_invalid_token_post(client): nr_proj_user_keys_before = models.ProjectUserKeys.query.count() - researchuser_pw_hash_before = ( - models.User.query.filter_by(username="researchuser").first()._password_hash + user = models.User.query.filter_by(username=researcher["username"]).first() + researchuser_pw_hash_before = user._password_hash + + # Add new row to password reset + new_reset_row = models.PasswordReset( + user=user.username, email=user.primary_email, issued=dds_cli.utils.timestamp() ) + db.session.add(new_reset_row) + db.session.commit() + # Need to use a valid token for the get request to get the form token - valid_reset_token = get_valid_reset_token("researchuser") + valid_reset_token = get_valid_reset_token(researcher["username"]) response = client.get( tests.DDSEndpoint.RESET_PASSWORD + valid_reset_token, follow_redirects=True ) From b4f1fc8a0bbeb3f91671a50390bc67d37af44976 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Tue, 15 Mar 2022 14:26:36 +0100 Subject: [PATCH 082/293] changed test --- tests/test_user_reset_password.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/test_user_reset_password.py b/tests/test_user_reset_password.py index cccacbb1e..673bf38a5 100644 --- a/tests/test_user_reset_password.py +++ b/tests/test_user_reset_password.py @@ -1,7 +1,6 @@ from cryptography.hazmat.primitives.ciphers.aead import AESGCM import datetime import http -import dds_cli import flask import flask_mail import pytest @@ -9,6 +8,7 @@ import tests from dds_web import db +from dds_web import utils from dds_web.database import models from dds_web.security.project_user_keys import generate_invite_key_pair from dds_web.security.tokens import encrypted_jwt_token @@ -123,7 +123,7 @@ def test_reset_password_invalid_token_post(client): # Add new row to password reset new_reset_row = models.PasswordReset( - user=user.username, email=user.primary_email, issued=dds_cli.utils.timestamp() + user=user, email=user.primary_email, issued=utils.timestamp() ) db.session.add(new_reset_row) db.session.commit() @@ -175,11 +175,18 @@ def test_reset_password_expired_token_get(client): def test_reset_password_expired_token_post(client): nr_proj_user_keys_before = models.ProjectUserKeys.query.count() - researchuser_pw_hash_before = ( - models.User.query.filter_by(username="researchuser").first()._password_hash + user = models.User.query.filter_by(username=researcher["username"]).first() + researchuser_pw_hash_before = user._password_hash + + # Add new row to password reset + new_reset_row = models.PasswordReset( + user=user, email=user.primary_email, issued=utils.timestamp() ) + db.session.add(new_reset_row) + db.session.commit() + # Need to use a valid token for the get request to get the form token - valid_reset_token = get_valid_reset_token("researchuser") + valid_reset_token = get_valid_reset_token(researcher["username"]) response = client.get( tests.DDSEndpoint.RESET_PASSWORD + valid_reset_token, follow_redirects=True ) From 5d78c5926b39ddc963897c971f43da85595bffdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Tue, 15 Mar 2022 15:12:54 +0100 Subject: [PATCH 083/293] fix tests --- tests/test_user_reset_password.py | 161 ++++++++++++++++++++++++++---- 1 file changed, 140 insertions(+), 21 deletions(-) diff --git a/tests/test_user_reset_password.py b/tests/test_user_reset_password.py index 673bf38a5..297527af5 100644 --- a/tests/test_user_reset_password.py +++ b/tests/test_user_reset_password.py @@ -13,7 +13,11 @@ from dds_web.security.project_user_keys import generate_invite_key_pair from dds_web.security.tokens import encrypted_jwt_token -researcher = {"username": "researchuser", "email": "researchuser@mailtrap.io"} +test_users = { + "researcher": {"username": "researchuser"}, + "projectowner": {"username": "projectowner"}, + "unituser": {"username": "unituser"}, +} def test_request_reset_password_no_form(client): @@ -48,7 +52,9 @@ def test_request_reset_password_inactive_user(client): assert response.status_code == http.HTTPStatus.OK form_token = flask.g.csrf_token - researchuser = models.User.query.filter_by(username="researchuser").first() + researchuser = models.User.query.filter_by( + username=test_users["researcher"]["username"] + ).first() researchuser.active = False db.session.add(researchuser) db.session.commit() @@ -95,7 +101,9 @@ def test_reset_password_no_token(client): def test_reset_password_invalid_token_get(client): - auth_token_header = tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client) + auth_token_header = tests.UserAuth( + tests.USER_CREDENTIALS[test_users["unituser"]["username"]] + ).token(client) token = auth_token_header["Authorization"].split(" ")[1] response = client.get(tests.DDSEndpoint.RESET_PASSWORD + token, follow_redirects=True) @@ -118,7 +126,7 @@ def get_valid_reset_token(username, expires_in=3600): def test_reset_password_invalid_token_post(client): nr_proj_user_keys_before = models.ProjectUserKeys.query.count() - user = models.User.query.filter_by(username=researcher["username"]).first() + user = models.User.query.filter_by(username=test_users["researcher"]["username"]).first() researchuser_pw_hash_before = user._password_hash # Add new row to password reset @@ -129,7 +137,7 @@ def test_reset_password_invalid_token_post(client): db.session.commit() # Need to use a valid token for the get request to get the form token - valid_reset_token = get_valid_reset_token(researcher["username"]) + valid_reset_token = get_valid_reset_token(test_users["researcher"]["username"]) response = client.get( tests.DDSEndpoint.RESET_PASSWORD + valid_reset_token, follow_redirects=True ) @@ -145,7 +153,9 @@ def test_reset_password_invalid_token_post(client): "submit": "Reset Password", } - auth_token_header = tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client) + auth_token_header = tests.UserAuth( + tests.USER_CREDENTIALS[test_users["unituser"]["username"]] + ).token(client) invalid_token = auth_token_header["Authorization"].split(" ")[1] response = client.post( @@ -159,13 +169,15 @@ def test_reset_password_invalid_token_post(client): assert nr_proj_user_keys_before == nr_proj_user_keys_after researchuser_pw_hash_after = ( - models.User.query.filter_by(username="researchuser").first()._password_hash + models.User.query.filter_by(username=test_users["researcher"]["username"]) + .first() + ._password_hash ) assert researchuser_pw_hash_before == researchuser_pw_hash_after def test_reset_password_expired_token_get(client): - token = get_valid_reset_token("researchuser", expires_in=-1) + token = get_valid_reset_token(test_users["researcher"]["username"], expires_in=-1) response = client.get(tests.DDSEndpoint.RESET_PASSWORD + token, follow_redirects=True) assert response.status_code == http.HTTPStatus.OK @@ -173,9 +185,24 @@ def test_reset_password_expired_token_get(client): assert flask.request.path == tests.DDSEndpoint.INDEX +def test_reset_password_expired_token_post_no_password_reset_row(client): + nr_proj_user_keys_before = models.ProjectUserKeys.query.count() + user = models.User.query.filter_by(username=test_users["researcher"]["username"]).first() + researchuser_pw_hash_before = user._password_hash + + # Need to use a valid token for the get request to get the form token + valid_reset_token = get_valid_reset_token(test_users["researcher"]["username"]) + response = client.get( + tests.DDSEndpoint.RESET_PASSWORD + valid_reset_token, follow_redirects=True + ) + + assert response.status_code == http.HTTPStatus.OK + assert flask.request.path == tests.DDSEndpoint.INDEX + + def test_reset_password_expired_token_post(client): nr_proj_user_keys_before = models.ProjectUserKeys.query.count() - user = models.User.query.filter_by(username=researcher["username"]).first() + user = models.User.query.filter_by(username=test_users["researcher"]["username"]).first() researchuser_pw_hash_before = user._password_hash # Add new row to password reset @@ -186,7 +213,7 @@ def test_reset_password_expired_token_post(client): db.session.commit() # Need to use a valid token for the get request to get the form token - valid_reset_token = get_valid_reset_token(researcher["username"]) + valid_reset_token = get_valid_reset_token(test_users["researcher"]["username"]) response = client.get( tests.DDSEndpoint.RESET_PASSWORD + valid_reset_token, follow_redirects=True ) @@ -202,7 +229,7 @@ def test_reset_password_expired_token_post(client): "submit": "Reset Password", } - expired_token = get_valid_reset_token("researchuser", expires_in=-1) + expired_token = get_valid_reset_token(test_users["researcher"]["username"], expires_in=-1) response = client.post( tests.DDSEndpoint.RESET_PASSWORD + expired_token, json=form_data, follow_redirects=True @@ -215,22 +242,54 @@ def test_reset_password_expired_token_post(client): assert nr_proj_user_keys_before == nr_proj_user_keys_after researchuser_pw_hash_after = ( - models.User.query.filter_by(username="researchuser").first()._password_hash + models.User.query.filter_by(username=test_users["researcher"]["username"]) + .first() + ._password_hash ) assert researchuser_pw_hash_before == researchuser_pw_hash_after +def test_reset_password_researchuser_no_password_reset_row(client): + user = models.User.query.filter_by(username=test_users["researcher"]["username"]).first() + nr_proj_user_keys_total_before = models.ProjectUserKeys.query.count() + assert nr_proj_user_keys_total_before > 0 + + nr_proj_user_keys_before = len(user.project_user_keys) + assert nr_proj_user_keys_before > 0 + + user_pw_hash_before = user._password_hash + user_public_key_before = user.public_key + + # Need to use a valid token for the get request to get the form token + valid_reset_token = get_valid_reset_token(test_users["researcher"]["username"]) + response = client.get( + tests.DDSEndpoint.RESET_PASSWORD + valid_reset_token, follow_redirects=True + ) + + assert response.status_code == http.HTTPStatus.OK + assert flask.request.path == tests.DDSEndpoint.INDEX + + def test_reset_password_researchuser(client): - user = models.User.query.filter_by(username="researchuser").first() + user = models.User.query.filter_by(username=test_users["researcher"]["username"]).first() nr_proj_user_keys_total_before = models.ProjectUserKeys.query.count() assert nr_proj_user_keys_total_before > 0 + nr_proj_user_keys_before = len(user.project_user_keys) assert nr_proj_user_keys_before > 0 + user_pw_hash_before = user._password_hash user_public_key_before = user.public_key + # Add new row to password reset + new_reset_row = models.PasswordReset( + user=user, email=user.primary_email, issued=utils.timestamp() + ) + db.session.add(new_reset_row) + db.session.commit() + # Need to use a valid token for the get request to get the form token - valid_reset_token = get_valid_reset_token("researchuser") + valid_reset_token = get_valid_reset_token(test_users["researcher"]["username"]) response = client.get( tests.DDSEndpoint.RESET_PASSWORD + valid_reset_token, follow_redirects=True ) @@ -252,7 +311,7 @@ def test_reset_password_researchuser(client): assert response.status_code == http.HTTPStatus.OK assert flask.request.path == tests.DDSEndpoint.PASSWORD_RESET_COMPLETED - user = models.User.query.filter_by(username="researchuser").first() + user = models.User.query.filter_by(username=test_users["researcher"]["username"]).first() # All users project keys should have been removed nr_proj_user_keys_after = len(user.project_user_keys) @@ -272,17 +331,47 @@ def test_reset_password_researchuser(client): assert user_public_key_before != user_public_key_after +def test_reset_password_project_owner_no_password_reset_row(client): + user = models.User.query.filter_by(username=test_users["projectowner"]["username"]).first() + nr_proj_user_keys_total_before = models.ProjectUserKeys.query.count() + assert nr_proj_user_keys_total_before > 0 + + nr_proj_user_keys_before = len(user.project_user_keys) + assert nr_proj_user_keys_before > 0 + + user_pw_hash_before = user._password_hash + user_public_key_before = user.public_key + + # Need to use a valid token for the get request to get the form token + valid_reset_token = get_valid_reset_token(test_users["projectowner"]["username"]) + response = client.get( + tests.DDSEndpoint.RESET_PASSWORD + valid_reset_token, follow_redirects=True + ) + + assert response.status_code == http.HTTPStatus.OK + assert flask.request.path == tests.DDSEndpoint.INDEX + + def test_reset_password_project_owner(client): - user = models.User.query.filter_by(username="projectowner").first() + user = models.User.query.filter_by(username=test_users["projectowner"]["username"]).first() nr_proj_user_keys_total_before = models.ProjectUserKeys.query.count() assert nr_proj_user_keys_total_before > 0 + nr_proj_user_keys_before = len(user.project_user_keys) assert nr_proj_user_keys_before > 0 + user_pw_hash_before = user._password_hash user_public_key_before = user.public_key + # Add new row to password reset + new_reset_row = models.PasswordReset( + user=user, email=user.primary_email, issued=utils.timestamp() + ) + db.session.add(new_reset_row) + db.session.commit() + # Need to use a valid token for the get request to get the form token - valid_reset_token = get_valid_reset_token("projectowner") + valid_reset_token = get_valid_reset_token(test_users["projectowner"]["username"]) response = client.get( tests.DDSEndpoint.RESET_PASSWORD + valid_reset_token, follow_redirects=True ) @@ -304,7 +393,7 @@ def test_reset_password_project_owner(client): assert response.status_code == http.HTTPStatus.OK assert flask.request.path == tests.DDSEndpoint.PASSWORD_RESET_COMPLETED - user = models.User.query.filter_by(username="projectowner").first() + user = models.User.query.filter_by(username=test_users["projectowner"]["username"]).first() # All users project keys should have been removed nr_proj_user_keys_after = len(user.project_user_keys) @@ -324,17 +413,47 @@ def test_reset_password_project_owner(client): assert user_public_key_before != user_public_key_after +def test_reset_password_unituser_no_password_reset_row(client): + user = models.User.query.filter_by(username=test_users["unituser"]["username"]).first() + nr_proj_user_keys_total_before = models.ProjectUserKeys.query.count() + assert nr_proj_user_keys_total_before > 0 + + nr_proj_user_keys_before = len(user.project_user_keys) + assert nr_proj_user_keys_before > 0 + + user_pw_hash_before = user._password_hash + user_public_key_before = user.public_key + + # Need to use a valid token for the get request to get the form token + valid_reset_token = get_valid_reset_token(test_users["unituser"]["username"]) + response = client.get( + tests.DDSEndpoint.RESET_PASSWORD + valid_reset_token, follow_redirects=True + ) + + assert response.status_code == http.HTTPStatus.OK + assert flask.request.path == tests.DDSEndpoint.INDEX + + def test_reset_password_unituser(client): - user = models.User.query.filter_by(username="unituser").first() + user = models.User.query.filter_by(username=test_users["unituser"]["username"]).first() nr_proj_user_keys_total_before = models.ProjectUserKeys.query.count() assert nr_proj_user_keys_total_before > 0 + nr_proj_user_keys_before = len(user.project_user_keys) assert nr_proj_user_keys_before > 0 + user_pw_hash_before = user._password_hash user_public_key_before = user.public_key + # Add new row to password reset + new_reset_row = models.PasswordReset( + user=user, email=user.primary_email, issued=utils.timestamp() + ) + db.session.add(new_reset_row) + db.session.commit() + # Need to use a valid token for the get request to get the form token - valid_reset_token = get_valid_reset_token("unituser") + valid_reset_token = get_valid_reset_token(test_users["unituser"]["username"]) response = client.get( tests.DDSEndpoint.RESET_PASSWORD + valid_reset_token, follow_redirects=True ) @@ -356,7 +475,7 @@ def test_reset_password_unituser(client): assert response.status_code == http.HTTPStatus.OK assert flask.request.path == tests.DDSEndpoint.PASSWORD_RESET_COMPLETED - user = models.User.query.filter_by(username="unituser").first() + user = models.User.query.filter_by(username=test_users["unituser"]["username"]).first() # All users project keys should have been removed nr_proj_user_keys_after = len(user.project_user_keys) From ba273fc430b87f66517790e271178739f6e09350 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Tue, 15 Mar 2022 16:09:19 +0100 Subject: [PATCH 084/293] changed test config --- tests/test_user_reset_password.py | 72 ++++++++++++------------------- 1 file changed, 28 insertions(+), 44 deletions(-) diff --git a/tests/test_user_reset_password.py b/tests/test_user_reset_password.py index 297527af5..537ca51eb 100644 --- a/tests/test_user_reset_password.py +++ b/tests/test_user_reset_password.py @@ -13,12 +13,6 @@ from dds_web.security.project_user_keys import generate_invite_key_pair from dds_web.security.tokens import encrypted_jwt_token -test_users = { - "researcher": {"username": "researchuser"}, - "projectowner": {"username": "projectowner"}, - "unituser": {"username": "unituser"}, -} - def test_request_reset_password_no_form(client): with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: @@ -52,9 +46,7 @@ def test_request_reset_password_inactive_user(client): assert response.status_code == http.HTTPStatus.OK form_token = flask.g.csrf_token - researchuser = models.User.query.filter_by( - username=test_users["researcher"]["username"] - ).first() + researchuser = models.User.query.filter_by(username="researchuser").first() researchuser.active = False db.session.add(researchuser) db.session.commit() @@ -101,9 +93,7 @@ def test_reset_password_no_token(client): def test_reset_password_invalid_token_get(client): - auth_token_header = tests.UserAuth( - tests.USER_CREDENTIALS[test_users["unituser"]["username"]] - ).token(client) + auth_token_header = tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client) token = auth_token_header["Authorization"].split(" ")[1] response = client.get(tests.DDSEndpoint.RESET_PASSWORD + token, follow_redirects=True) @@ -126,7 +116,7 @@ def get_valid_reset_token(username, expires_in=3600): def test_reset_password_invalid_token_post(client): nr_proj_user_keys_before = models.ProjectUserKeys.query.count() - user = models.User.query.filter_by(username=test_users["researcher"]["username"]).first() + user = models.User.query.filter_by(username="researchuser").first() researchuser_pw_hash_before = user._password_hash # Add new row to password reset @@ -137,7 +127,7 @@ def test_reset_password_invalid_token_post(client): db.session.commit() # Need to use a valid token for the get request to get the form token - valid_reset_token = get_valid_reset_token(test_users["researcher"]["username"]) + valid_reset_token = get_valid_reset_token("researchuser") response = client.get( tests.DDSEndpoint.RESET_PASSWORD + valid_reset_token, follow_redirects=True ) @@ -153,9 +143,7 @@ def test_reset_password_invalid_token_post(client): "submit": "Reset Password", } - auth_token_header = tests.UserAuth( - tests.USER_CREDENTIALS[test_users["unituser"]["username"]] - ).token(client) + auth_token_header = tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client) invalid_token = auth_token_header["Authorization"].split(" ")[1] response = client.post( @@ -169,15 +157,13 @@ def test_reset_password_invalid_token_post(client): assert nr_proj_user_keys_before == nr_proj_user_keys_after researchuser_pw_hash_after = ( - models.User.query.filter_by(username=test_users["researcher"]["username"]) - .first() - ._password_hash + models.User.query.filter_by(username="researchuser").first()._password_hash ) assert researchuser_pw_hash_before == researchuser_pw_hash_after def test_reset_password_expired_token_get(client): - token = get_valid_reset_token(test_users["researcher"]["username"], expires_in=-1) + token = get_valid_reset_token("researchuser", expires_in=-1) response = client.get(tests.DDSEndpoint.RESET_PASSWORD + token, follow_redirects=True) assert response.status_code == http.HTTPStatus.OK @@ -187,11 +173,11 @@ def test_reset_password_expired_token_get(client): def test_reset_password_expired_token_post_no_password_reset_row(client): nr_proj_user_keys_before = models.ProjectUserKeys.query.count() - user = models.User.query.filter_by(username=test_users["researcher"]["username"]).first() + user = models.User.query.filter_by(username="researchuser").first() researchuser_pw_hash_before = user._password_hash # Need to use a valid token for the get request to get the form token - valid_reset_token = get_valid_reset_token(test_users["researcher"]["username"]) + valid_reset_token = get_valid_reset_token("researchuser") response = client.get( tests.DDSEndpoint.RESET_PASSWORD + valid_reset_token, follow_redirects=True ) @@ -202,7 +188,7 @@ def test_reset_password_expired_token_post_no_password_reset_row(client): def test_reset_password_expired_token_post(client): nr_proj_user_keys_before = models.ProjectUserKeys.query.count() - user = models.User.query.filter_by(username=test_users["researcher"]["username"]).first() + user = models.User.query.filter_by(username="researchuser").first() researchuser_pw_hash_before = user._password_hash # Add new row to password reset @@ -213,7 +199,7 @@ def test_reset_password_expired_token_post(client): db.session.commit() # Need to use a valid token for the get request to get the form token - valid_reset_token = get_valid_reset_token(test_users["researcher"]["username"]) + valid_reset_token = get_valid_reset_token("researchuser") response = client.get( tests.DDSEndpoint.RESET_PASSWORD + valid_reset_token, follow_redirects=True ) @@ -229,7 +215,7 @@ def test_reset_password_expired_token_post(client): "submit": "Reset Password", } - expired_token = get_valid_reset_token(test_users["researcher"]["username"], expires_in=-1) + expired_token = get_valid_reset_token("researchuser", expires_in=-1) response = client.post( tests.DDSEndpoint.RESET_PASSWORD + expired_token, json=form_data, follow_redirects=True @@ -242,15 +228,13 @@ def test_reset_password_expired_token_post(client): assert nr_proj_user_keys_before == nr_proj_user_keys_after researchuser_pw_hash_after = ( - models.User.query.filter_by(username=test_users["researcher"]["username"]) - .first() - ._password_hash + models.User.query.filter_by(username="researchuser").first()._password_hash ) assert researchuser_pw_hash_before == researchuser_pw_hash_after def test_reset_password_researchuser_no_password_reset_row(client): - user = models.User.query.filter_by(username=test_users["researcher"]["username"]).first() + user = models.User.query.filter_by(username="researchuser").first() nr_proj_user_keys_total_before = models.ProjectUserKeys.query.count() assert nr_proj_user_keys_total_before > 0 @@ -261,7 +245,7 @@ def test_reset_password_researchuser_no_password_reset_row(client): user_public_key_before = user.public_key # Need to use a valid token for the get request to get the form token - valid_reset_token = get_valid_reset_token(test_users["researcher"]["username"]) + valid_reset_token = get_valid_reset_token("researchuser") response = client.get( tests.DDSEndpoint.RESET_PASSWORD + valid_reset_token, follow_redirects=True ) @@ -271,7 +255,7 @@ def test_reset_password_researchuser_no_password_reset_row(client): def test_reset_password_researchuser(client): - user = models.User.query.filter_by(username=test_users["researcher"]["username"]).first() + user = models.User.query.filter_by(username="researchuser").first() nr_proj_user_keys_total_before = models.ProjectUserKeys.query.count() assert nr_proj_user_keys_total_before > 0 @@ -289,7 +273,7 @@ def test_reset_password_researchuser(client): db.session.commit() # Need to use a valid token for the get request to get the form token - valid_reset_token = get_valid_reset_token(test_users["researcher"]["username"]) + valid_reset_token = get_valid_reset_token("researchuser") response = client.get( tests.DDSEndpoint.RESET_PASSWORD + valid_reset_token, follow_redirects=True ) @@ -311,7 +295,7 @@ def test_reset_password_researchuser(client): assert response.status_code == http.HTTPStatus.OK assert flask.request.path == tests.DDSEndpoint.PASSWORD_RESET_COMPLETED - user = models.User.query.filter_by(username=test_users["researcher"]["username"]).first() + user = models.User.query.filter_by(username="researchuser").first() # All users project keys should have been removed nr_proj_user_keys_after = len(user.project_user_keys) @@ -332,7 +316,7 @@ def test_reset_password_researchuser(client): def test_reset_password_project_owner_no_password_reset_row(client): - user = models.User.query.filter_by(username=test_users["projectowner"]["username"]).first() + user = models.User.query.filter_by(username="projectowner").first() nr_proj_user_keys_total_before = models.ProjectUserKeys.query.count() assert nr_proj_user_keys_total_before > 0 @@ -343,7 +327,7 @@ def test_reset_password_project_owner_no_password_reset_row(client): user_public_key_before = user.public_key # Need to use a valid token for the get request to get the form token - valid_reset_token = get_valid_reset_token(test_users["projectowner"]["username"]) + valid_reset_token = get_valid_reset_token("projectowner") response = client.get( tests.DDSEndpoint.RESET_PASSWORD + valid_reset_token, follow_redirects=True ) @@ -353,7 +337,7 @@ def test_reset_password_project_owner_no_password_reset_row(client): def test_reset_password_project_owner(client): - user = models.User.query.filter_by(username=test_users["projectowner"]["username"]).first() + user = models.User.query.filter_by(username="projectowner").first() nr_proj_user_keys_total_before = models.ProjectUserKeys.query.count() assert nr_proj_user_keys_total_before > 0 @@ -371,7 +355,7 @@ def test_reset_password_project_owner(client): db.session.commit() # Need to use a valid token for the get request to get the form token - valid_reset_token = get_valid_reset_token(test_users["projectowner"]["username"]) + valid_reset_token = get_valid_reset_token("projectowner") response = client.get( tests.DDSEndpoint.RESET_PASSWORD + valid_reset_token, follow_redirects=True ) @@ -393,7 +377,7 @@ def test_reset_password_project_owner(client): assert response.status_code == http.HTTPStatus.OK assert flask.request.path == tests.DDSEndpoint.PASSWORD_RESET_COMPLETED - user = models.User.query.filter_by(username=test_users["projectowner"]["username"]).first() + user = models.User.query.filter_by(username="projectowner").first() # All users project keys should have been removed nr_proj_user_keys_after = len(user.project_user_keys) @@ -414,7 +398,7 @@ def test_reset_password_project_owner(client): def test_reset_password_unituser_no_password_reset_row(client): - user = models.User.query.filter_by(username=test_users["unituser"]["username"]).first() + user = models.User.query.filter_by(username="unituser").first() nr_proj_user_keys_total_before = models.ProjectUserKeys.query.count() assert nr_proj_user_keys_total_before > 0 @@ -425,7 +409,7 @@ def test_reset_password_unituser_no_password_reset_row(client): user_public_key_before = user.public_key # Need to use a valid token for the get request to get the form token - valid_reset_token = get_valid_reset_token(test_users["unituser"]["username"]) + valid_reset_token = get_valid_reset_token("unituser") response = client.get( tests.DDSEndpoint.RESET_PASSWORD + valid_reset_token, follow_redirects=True ) @@ -435,7 +419,7 @@ def test_reset_password_unituser_no_password_reset_row(client): def test_reset_password_unituser(client): - user = models.User.query.filter_by(username=test_users["unituser"]["username"]).first() + user = models.User.query.filter_by(username="unituser").first() nr_proj_user_keys_total_before = models.ProjectUserKeys.query.count() assert nr_proj_user_keys_total_before > 0 @@ -453,7 +437,7 @@ def test_reset_password_unituser(client): db.session.commit() # Need to use a valid token for the get request to get the form token - valid_reset_token = get_valid_reset_token(test_users["unituser"]["username"]) + valid_reset_token = get_valid_reset_token("unituser") response = client.get( tests.DDSEndpoint.RESET_PASSWORD + valid_reset_token, follow_redirects=True ) @@ -475,7 +459,7 @@ def test_reset_password_unituser(client): assert response.status_code == http.HTTPStatus.OK assert flask.request.path == tests.DDSEndpoint.PASSWORD_RESET_COMPLETED - user = models.User.query.filter_by(username=test_users["unituser"]["username"]).first() + user = models.User.query.filter_by(username="unituser").first() # All users project keys should have been removed nr_proj_user_keys_after = len(user.project_user_keys) From b6c8ba01629f1a55aafb4d85798863a4028b11a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Tue, 15 Mar 2022 17:52:40 +0100 Subject: [PATCH 085/293] List users in another unit with superadmin --- dds_web/api/user.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index b1bcdafe8..9b402f260 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -935,7 +935,7 @@ def get(self): class UnitUsers(flask_restful.Resource): """List unit users.""" - @auth.login_required(role=["Unit Admin", "Unit Personnel"]) + @auth.login_required(role=["Super Admin", "Unit Admin", "Unit Personnel"]) @logging_bind_request def get(self): """Get and return unit users within the unit the current user is connected to.""" @@ -949,6 +949,27 @@ def get(self): ) ) + if auth.current_user().role == "Super Admin": + json_input = flask.request.json + if not json_input: + raise ddserr.DDSArgumentError(message="Unit public id missing.") + + unit = json_input.get("unit") + if not unit: + raise ddserr.DDSArgumentError(message="Unit public id missing.") + + unit_row = models.Unit.query.filter_by(public_id=unit).one_or_none() + if not unit_row: + raise ddserr.DDSArgumentError( + message=f"There is no unit with the public id '{unit}'." + ) + else: + unit_row = auth.current_user().unit + + return self.get_unit_users(unit=unit_row) + + def get_unit_users(self, unit): + """Get users within a specific unit.""" keys = ["Name", "Username", "Email", "Role", "Active"] unit_users = [ { @@ -958,7 +979,7 @@ def get(self): "Role": user.role, "Active": user.is_active, } - for user in auth.current_user().unit.users + for user in unit.users ] - return {"users": unit_users, "keys": keys, "unit": auth.current_user().unit.name} + return {"users": unit_users, "keys": keys, "unit": unit.name} From a456374e9a245fb0b2a94e303162798c5a9c45a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Tue, 15 Mar 2022 20:15:09 +0100 Subject: [PATCH 086/293] super admin tests --- dds_web/api/user.py | 8 ++---- tests/test_users_list.py | 62 +++++++++++++++++++++++++++++++++++----- 2 files changed, 57 insertions(+), 13 deletions(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index 9b402f260..6674581c8 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -966,10 +966,6 @@ def get(self): else: unit_row = auth.current_user().unit - return self.get_unit_users(unit=unit_row) - - def get_unit_users(self, unit): - """Get users within a specific unit.""" keys = ["Name", "Username", "Email", "Role", "Active"] unit_users = [ { @@ -979,7 +975,7 @@ def get_unit_users(self, unit): "Role": user.role, "Active": user.is_active, } - for user in unit.users + for user in unit_row.users ] - return {"users": unit_users, "keys": keys, "unit": unit.name} + return {"users": unit_users, "keys": keys, "unit": unit_row.name} diff --git a/tests/test_users_list.py b/tests/test_users_list.py index 588a4f334..a81fbf07f 100644 --- a/tests/test_users_list.py +++ b/tests/test_users_list.py @@ -32,13 +32,6 @@ def test_list_unitusers_with_researcher(client): assert response.status_code == http.HTTPStatus.FORBIDDEN -def test_list_unitusers_with_super_admin(client): - """Super admins will be able to list unit users, but not right now.""" - token = get_token(username=users["Super Admin"], client=client) - response = client.get(tests.DDSEndpoint.LIST_UNIT_USERS, headers=token) - assert response.status_code == http.HTTPStatus.FORBIDDEN - - def test_list_unitusers_with_unit_personnel_and_admin_deactivated(client): """Unit Personnel should be able to list the users within a unit.""" # Deactivate user @@ -88,3 +81,58 @@ def test_list_unitusers_with_unit_personnel_and_admin_ok(client): "Active": dbrow.is_active, } assert expected in users_in_response + + +def test_list_unitusers_with_super_admin_no_unit(client): + """Super admins need to specify a unit.""" + token = get_token(username=users["Super Admin"], client=client) + response = client.get(tests.DDSEndpoint.LIST_UNIT_USERS, headers=token) + assert response.status_code == http.HTTPStatus.BAD_REQUEST + assert "Unit public id missing" in response.json.get("message") + + +def test_list_unitusers_with_super_admin_unit_empty(client): + """Super admins need to specify a unit.""" + token = get_token(username=users["Super Admin"], client=client) + for x in [None, ""]: + response = client.get(tests.DDSEndpoint.LIST_UNIT_USERS, json={"unit": x}, headers=token) + assert response.status_code == http.HTTPStatus.BAD_REQUEST + assert "Unit public id missing" in response.json.get("message") + + +def test_list_unitusers_with_super_admin_nonexistent_unit(client): + """Super admins need to specify a correct unit.""" + incorrect_unit = "incorrect_unit" + token = get_token(username=users["Super Admin"], client=client) + response = client.get( + tests.DDSEndpoint.LIST_UNIT_USERS, json={"unit": incorrect_unit}, headers=token + ) + assert response.status_code == http.HTTPStatus.BAD_REQUEST + assert f"There is no unit with the public id '{incorrect_unit}'" in response.json.get("message") + + +def test_list_unitusers_with_super_admin_correct_unit(client): + """Super admins can list users in a specific unit.""" + unit_row = models.Unit.query.filter_by(name="Unit 1").one_or_none() + assert unit_row + + token = get_token( + username=users["Super Admin"], json={"unit": unit_row.public_id}, client=client + ) + response = client.get(tests.DDSEndpoint.LIST_UNIT_USERS, headers=token) + assert response.status_code == http.HTTPStatus.OK + + assert all(x in response.json for x in ["users", "keys", "unit"]) + + returned_users = response.json.get("users") + returned_keys = response.json.get("keys") + returned_unit = response.json.get("unit") + + assert returned_users and returned_keys and returned_unit + assert returned_keys == ["Name", "Username", "Email", "Role", "Active"] + assert returned_unit == unit_row.name + assert len(unit_users) == len(unit_row.users) + + unit_users = [x["Username"] for x in returned_users] + for x in unit_row.users: + assert x.username in unit_users From bece1081f14ee4da29ae7f0ca80b8931adf03d59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Tue, 15 Mar 2022 21:41:39 +0100 Subject: [PATCH 087/293] new unit module --- dds_web/api/__init__.py | 5 +++ dds_web/api/unit.py | 84 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 dds_web/api/unit.py diff --git a/dds_web/api/__init__.py b/dds_web/api/__init__.py index 0d84495f0..e0ccc2880 100644 --- a/dds_web/api/__init__.py +++ b/dds_web/api/__init__.py @@ -13,6 +13,7 @@ from dds_web.api import project from dds_web.api import s3 from dds_web.api import files +from dds_web.api import unit #################################################################################################### # BLUEPRINTS ########################################################################## BLUEPRINTS # @@ -69,5 +70,9 @@ def output_json(data, code, headers=None): api.add_resource(user.UserActivation, "/user/activation", endpoint="user_activation") api.add_resource(user.UnitUsers, "/unit/users", endpoint="unit_users") +# Units #################################################################################### Units # + +api.add_resource(unit.AllUnits, "/unit/info/all", endpoint="all_units") + # Invoicing ############################################################################ Invoicing # api.add_resource(user.ShowUsage, "/usage", endpoint="usage") diff --git a/dds_web/api/unit.py b/dds_web/api/unit.py new file mode 100644 index 000000000..7cc4d70c5 --- /dev/null +++ b/dds_web/api/unit.py @@ -0,0 +1,84 @@ +"""User related endpoints e.g. authentication.""" + +#################################################################################################### +# IMPORTS ################################################################################ IMPORTS # +#################################################################################################### + +# Standard library +import os +import smtplib +import time +import datetime + +# Installed +import flask +import flask_restful +import flask_mail +import itsdangerous +import structlog +import sqlalchemy +import http + + +# Own modules +from dds_web import auth, mail, db, basic_auth, limiter +from dds_web.database import models +import dds_web.utils +import dds_web.forms +import dds_web.errors as ddserr +from dds_web.api.schemas import project_schemas, user_schemas, token_schemas +from dds_web.api.dds_decorators import ( + logging_bind_request, + json_required, + handle_validation_errors, +) +from dds_web.security.project_user_keys import ( + generate_invite_key_pair, + share_project_private_key, +) +from dds_web.security.tokens import encrypted_jwt_token, update_token_with_mfa +from dds_web.security.auth import get_user_roles_common + + +# initiate bound logger +action_logger = structlog.getLogger("actions") + +#################################################################################################### +# ENDPOINTS ############################################################################ ENDPOINTS # +#################################################################################################### + + +class AllUnits(flask_restful.Resource): + """Get unit info.""" + + @auth.login_required(role=["Super Admin"]) + @logging_bind_request + def get(self): + """Return info about unit to super admin.""" + all_units = models.Unit.query.all() + + unit_info = [ + { + "Name": u.name, + "Public ID": u.public_id, + "External Display Name": u.external_display_name, + "Contact Email": u.contact_email, + "Safespring Endpoint": u.safespring_endpoint, + "Days In Available": u.days_in_available, + "Days In Expired": u.days_in_expired, + } + for u in all_units + ] + + return { + "units": unit_info, + "keys": [ + "Name", + "Public ID", + "External Display Name", + "Days In Available", + "Days In Expired", + "Safespring Endpoint", + "Contact Email", + ], + } From 0f9277665b9b85888d22340b90e5b281d96de000 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Tue, 15 Mar 2022 22:05:04 +0100 Subject: [PATCH 088/293] moved json to correct position --- tests/test_users_list.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_users_list.py b/tests/test_users_list.py index a81fbf07f..0892adffc 100644 --- a/tests/test_users_list.py +++ b/tests/test_users_list.py @@ -116,10 +116,10 @@ def test_list_unitusers_with_super_admin_correct_unit(client): unit_row = models.Unit.query.filter_by(name="Unit 1").one_or_none() assert unit_row - token = get_token( - username=users["Super Admin"], json={"unit": unit_row.public_id}, client=client + token = get_token(username=users["Super Admin"], client=client) + response = client.get( + tests.DDSEndpoint.LIST_UNIT_USERS, json={"unit": unit_row.public_id}, headers=token ) - response = client.get(tests.DDSEndpoint.LIST_UNIT_USERS, headers=token) assert response.status_code == http.HTTPStatus.OK assert all(x in response.json for x in ["users", "keys", "unit"]) From ae8660a20b85558187589ef5857603fdbf2f946c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Tue, 15 Mar 2022 22:08:41 +0100 Subject: [PATCH 089/293] removed unused imports --- dds_web/api/unit.py | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/dds_web/api/unit.py b/dds_web/api/unit.py index 7cc4d70c5..ba9909b69 100644 --- a/dds_web/api/unit.py +++ b/dds_web/api/unit.py @@ -5,39 +5,15 @@ #################################################################################################### # Standard library -import os -import smtplib -import time -import datetime # Installed -import flask import flask_restful -import flask_mail -import itsdangerous import structlog -import sqlalchemy -import http - # Own modules -from dds_web import auth, mail, db, basic_auth, limiter +from dds_web import auth from dds_web.database import models -import dds_web.utils -import dds_web.forms -import dds_web.errors as ddserr -from dds_web.api.schemas import project_schemas, user_schemas, token_schemas -from dds_web.api.dds_decorators import ( - logging_bind_request, - json_required, - handle_validation_errors, -) -from dds_web.security.project_user_keys import ( - generate_invite_key_pair, - share_project_private_key, -) -from dds_web.security.tokens import encrypted_jwt_token, update_token_with_mfa -from dds_web.security.auth import get_user_roles_common +from dds_web.api.dds_decorators import logging_bind_request # initiate bound logger From add52204900acb2bbc21036a8d4dedba6c2be875 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Tue, 15 Mar 2022 22:18:38 +0100 Subject: [PATCH 090/293] added tests for listing units --- tests/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/__init__.py b/tests/__init__.py index 37e260d1c..7f604d236 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -194,4 +194,7 @@ class DDSEndpoint: USAGE = BASE_ENDPOINT + "/usage" INVOICE = BASE_ENDPOINT + "/invoice" + # Units + LIST_UNITS_ALL = BASE_ENDPOINT + "/unit/info/all" + TIMEOUT = 5 From 25109463800f2245d6304d872aff9c95523cb25a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Tue, 15 Mar 2022 22:20:13 +0100 Subject: [PATCH 091/293] add tests for listing units --- tests/test_unit_list.py | 72 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 tests/test_unit_list.py diff --git a/tests/test_unit_list.py b/tests/test_unit_list.py new file mode 100644 index 000000000..79a62b6d4 --- /dev/null +++ b/tests/test_unit_list.py @@ -0,0 +1,72 @@ +# IMPORTS ################################################################################ IMPORTS # + +# Standard library +import http + +# Own +from dds_web import db +from dds_web.api import user +from dds_web.database import models +import tests + +# CONFIG ################################################################################## CONFIG # + +users = { + "Researcher": "researchuser", + "Unit Personnel": "unituser", + "Unit Admin": "unitadmin", + "Super Admin": "superadmin", +} + +# TESTS #################################################################################### TESTS # + + +def get_token(username, client): + return tests.UserAuth(tests.USER_CREDENTIALS[username]).token(client) + + +def list_units_as_not_superadmin(client): + """Only Super Admin can list users.""" + no_access_users = users.copy() + no_access_users.pop("Super Admin") + + for u in no_access_users: + token = get_token(username=users[u], client=client) + response = client.get(tests.DDSEndpoint.LIST_UNITS_ALL, headers=token) + assert response.status_code == http.HTTPStatus.UNAUTHORIZED + + +def list_units_as_super_admin(client): + """List units as Super Admin.""" + all_units = models.Unit.query.all() + + token = get_token(username=users["Super Admin"], client=client) + response = client.get(tests.DDSEndpoint.LIST_UNITS_ALL, headers=token) + assert response.status_code == http.HTTPStatus.OK + + keys = response.json.get("keys") + units = response.json.get("units") + assert keys and units + + assert keys == [ + "Name", + "Public ID", + "External Display Name", + "Days In Available", + "Days In Expired", + "Safespring Endpoint", + "Contact Email", + ] + assert len(all_units) == len(units) + + for unit in all_units: + expected = { + "Name": unit.name, + "Public ID": unit.public_id, + "External Display Name": unit.external_display_name, + "Contact Email": unit.contact_email, + "Safespring Endpoint": unit.safespring_endpoint, + "Days In Available": unit.days_in_available, + "Days In Expired": unit.days_in_expired, + } + assert expected in units From 8022a880d57f77dfceffc3931f24edea75d1b073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Tue, 15 Mar 2022 22:21:20 +0100 Subject: [PATCH 092/293] wrong variable name --- tests/test_users_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_users_list.py b/tests/test_users_list.py index 0892adffc..bb608e7b3 100644 --- a/tests/test_users_list.py +++ b/tests/test_users_list.py @@ -131,7 +131,7 @@ def test_list_unitusers_with_super_admin_correct_unit(client): assert returned_users and returned_keys and returned_unit assert returned_keys == ["Name", "Username", "Email", "Role", "Active"] assert returned_unit == unit_row.name - assert len(unit_users) == len(unit_row.users) + assert len(returned_users) == len(unit_row.users) unit_users = [x["Username"] for x in returned_users] for x in unit_row.users: From 4839ce10f8ccfc0ead98b89efb4e2db0d6559d4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Tue, 15 Mar 2022 22:23:56 +0100 Subject: [PATCH 093/293] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28c233724..aafbf28dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,3 +50,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - When listing the projects, return whether or not the user has a project key for that particular project ([#1049](https://github.com/ScilifelabDataCentre/dds_web/pull/1049)) - New endpoint for Unit Personnel and Admins to list the other Unit Personnel / Admins within their project ([#1050](https://github.com/ScilifelabDataCentre/dds_web/pull/1050)) - Make previous HOTP invalid at password reset ([#1054](https://github.com/ScilifelabDataCentre/dds_web/pull/1054)) +- New endpoint for listing unit users as Super Admin () \ No newline at end of file From 42dc667df91055bd45449833a030605b5170872a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Tue, 15 Mar 2022 22:24:38 +0100 Subject: [PATCH 094/293] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28c233724..e8f89b8a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,3 +50,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - When listing the projects, return whether or not the user has a project key for that particular project ([#1049](https://github.com/ScilifelabDataCentre/dds_web/pull/1049)) - New endpoint for Unit Personnel and Admins to list the other Unit Personnel / Admins within their project ([#1050](https://github.com/ScilifelabDataCentre/dds_web/pull/1050)) - Make previous HOTP invalid at password reset ([#1054](https://github.com/ScilifelabDataCentre/dds_web/pull/1054)) +- New endpoint for listing Units as Super Admin () \ No newline at end of file From 60824d43c1ae27a4b194840b2b7e792ad7bb603e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Tue, 15 Mar 2022 22:29:18 +0100 Subject: [PATCH 095/293] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aafbf28dc..86f3d1a15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,4 +50,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - When listing the projects, return whether or not the user has a project key for that particular project ([#1049](https://github.com/ScilifelabDataCentre/dds_web/pull/1049)) - New endpoint for Unit Personnel and Admins to list the other Unit Personnel / Admins within their project ([#1050](https://github.com/ScilifelabDataCentre/dds_web/pull/1050)) - Make previous HOTP invalid at password reset ([#1054](https://github.com/ScilifelabDataCentre/dds_web/pull/1054)) -- New endpoint for listing unit users as Super Admin () \ No newline at end of file +- New endpoint for listing unit users as Super Admin ([#1059](https://github.com/ScilifelabDataCentre/dds_web/pull/1059)) \ No newline at end of file From 475ea85ad6e69a542e0febe1793e16746ae813a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Tue, 15 Mar 2022 22:29:55 +0100 Subject: [PATCH 096/293] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8f89b8a4..374492237 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,4 +50,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - When listing the projects, return whether or not the user has a project key for that particular project ([#1049](https://github.com/ScilifelabDataCentre/dds_web/pull/1049)) - New endpoint for Unit Personnel and Admins to list the other Unit Personnel / Admins within their project ([#1050](https://github.com/ScilifelabDataCentre/dds_web/pull/1050)) - Make previous HOTP invalid at password reset ([#1054](https://github.com/ScilifelabDataCentre/dds_web/pull/1054)) -- New endpoint for listing Units as Super Admin () \ No newline at end of file +- New endpoint for listing Units as Super Admin ([1060](https://github.com/ScilifelabDataCentre/dds_web/pull/1060)) \ No newline at end of file From c75df7ec8d66934977a528b644ea55bf27241ad7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Tue, 15 Mar 2022 22:30:40 +0100 Subject: [PATCH 097/293] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86f3d1a15..e400122d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,4 +50,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - When listing the projects, return whether or not the user has a project key for that particular project ([#1049](https://github.com/ScilifelabDataCentre/dds_web/pull/1049)) - New endpoint for Unit Personnel and Admins to list the other Unit Personnel / Admins within their project ([#1050](https://github.com/ScilifelabDataCentre/dds_web/pull/1050)) - Make previous HOTP invalid at password reset ([#1054](https://github.com/ScilifelabDataCentre/dds_web/pull/1054)) -- New endpoint for listing unit users as Super Admin ([#1059](https://github.com/ScilifelabDataCentre/dds_web/pull/1059)) \ No newline at end of file +- New endpoint for listing unit users as Super Admin ([#1059](https://github.com/ScilifelabDataCentre/dds_web/pull/1059)) From 78a00f1a131bc1b02899b6ead493e6d1c6c2946e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Tue, 15 Mar 2022 22:31:02 +0100 Subject: [PATCH 098/293] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 374492237..6113f6e53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,4 +50,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - When listing the projects, return whether or not the user has a project key for that particular project ([#1049](https://github.com/ScilifelabDataCentre/dds_web/pull/1049)) - New endpoint for Unit Personnel and Admins to list the other Unit Personnel / Admins within their project ([#1050](https://github.com/ScilifelabDataCentre/dds_web/pull/1050)) - Make previous HOTP invalid at password reset ([#1054](https://github.com/ScilifelabDataCentre/dds_web/pull/1054)) -- New endpoint for listing Units as Super Admin ([1060](https://github.com/ScilifelabDataCentre/dds_web/pull/1060)) \ No newline at end of file +- New endpoint for listing Units as Super Admin ([1060](https://github.com/ScilifelabDataCentre/dds_web/pull/1060)) From ee789ff26abce9a01d2ef03537de00b7f3580ba7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Wed, 16 Mar 2022 07:35:59 +0100 Subject: [PATCH 099/293] the active check is not needed since it already exists in the authentication --- dds_web/api/user.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index b1bcdafe8..f8347f35f 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -941,14 +941,6 @@ def get(self): """Get and return unit users within the unit the current user is connected to.""" unit_users = {} - if not auth.current_user().is_active: - raise ddserr.AccessDeniedError( - message=( - "Your account has been deactivated. " - "You cannot list the users within your unit." - ) - ) - keys = ["Name", "Username", "Email", "Role", "Active"] unit_users = [ { From 94c14abd12ffb0476e74f18fb1dd3b79a0112c65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Wed, 16 Mar 2022 07:45:41 +0100 Subject: [PATCH 100/293] add decorator to handle sqlerrors during get requests --- dds_web/api/dds_decorators.py | 15 +++++++++++++++ dds_web/api/user.py | 4 +++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/dds_web/api/dds_decorators.py b/dds_web/api/dds_decorators.py index fbd78ae0b..0a7576c14 100644 --- a/dds_web/api/dds_decorators.py +++ b/dds_web/api/dds_decorators.py @@ -101,6 +101,21 @@ def make_commit(*args, **kwargs): return make_commit +def handle_db_error(func): + @functools.wraps(func) + def perform_get(*args, **kwargs): + + # Run function, catch errors + try: + result = func(*args, **kwargs) + except sqlalchemy.exc.SQLAlchemyError as sqlerr: + raise DatabaseError(message=str(sqlerr)) from sqlerr + + return result + + return perform_get + + # S3 ########################################################################################## S3 # diff --git a/dds_web/api/user.py b/dds_web/api/user.py index f8347f35f..9cf18a991 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -31,6 +31,7 @@ logging_bind_request, json_required, handle_validation_errors, + handle_db_error, ) from dds_web.security.project_user_keys import ( generate_invite_key_pair, @@ -937,11 +938,12 @@ class UnitUsers(flask_restful.Resource): @auth.login_required(role=["Unit Admin", "Unit Personnel"]) @logging_bind_request + @handle_db_error def get(self): """Get and return unit users within the unit the current user is connected to.""" unit_users = {} - keys = ["Name", "Username", "Email", "Role", "Active"] + unit_users = [ { "Name": user.name, From 4ecc8671302c192280e477ebb9e30a7b3d1a23a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Wed, 16 Mar 2022 07:47:49 +0100 Subject: [PATCH 101/293] add decorator to unit info too --- dds_web/api/unit.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dds_web/api/unit.py b/dds_web/api/unit.py index ba9909b69..07737c307 100644 --- a/dds_web/api/unit.py +++ b/dds_web/api/unit.py @@ -13,7 +13,7 @@ # Own modules from dds_web import auth from dds_web.database import models -from dds_web.api.dds_decorators import logging_bind_request +from dds_web.api.dds_decorators import logging_bind_request, handle_db_error # initiate bound logger @@ -29,6 +29,7 @@ class AllUnits(flask_restful.Resource): @auth.login_required(role=["Super Admin"]) @logging_bind_request + @handle_db_error def get(self): """Return info about unit to super admin.""" all_units = models.Unit.query.all() From 7b77aa07ed9cdb9501b46f4e04e82a59b1e22397 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Wed, 16 Mar 2022 09:25:59 +0100 Subject: [PATCH 102/293] changed the token expiration --- dds_web/security/auth.py | 2 +- dds_web/security/tokens.py | 6 +++--- tests/test_token.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dds_web/security/auth.py b/dds_web/security/auth.py index 84c3cda93..b56ef91a5 100644 --- a/dds_web/security/auth.py +++ b/dds_web/security/auth.py @@ -26,7 +26,7 @@ # VARIABLES ############################################################################ VARIABLES # -MFA_EXPIRES_IN = datetime.timedelta(hours=48) +MFA_EXPIRES_IN = datetime.timedelta(hours=168) #################################################################################################### # FUNCTIONS ############################################################################ FUNCTIONS # diff --git a/dds_web/security/tokens.py b/dds_web/security/tokens.py index fb8fe3f19..0d4294dd7 100644 --- a/dds_web/security/tokens.py +++ b/dds_web/security/tokens.py @@ -19,7 +19,7 @@ def encrypted_jwt_token( username, sensitive_content, - expires_in=datetime.timedelta(hours=48), + expires_in=datetime.timedelta(hours=168), additional_claims=None, fully_authenticated=False, ): @@ -76,7 +76,7 @@ def update_token_with_mfa(token_claims): def __signed_jwt_token( username, sensitive_content=None, - expires_in=datetime.timedelta(hours=48), + expires_in=datetime.timedelta(hours=168), additional_claims=None, ): """ @@ -104,7 +104,7 @@ def __signed_jwt_token( return token.serialize() -def jwt_token(username, expires_in=datetime.timedelta(hours=48), additional_claims=None): +def jwt_token(username, expires_in=datetime.timedelta(hours=168), additional_claims=None): """ Generates a signed JWT token. This is to be used for general purpose signed token. :param str username: Username must be obtained through authentication diff --git a/tests/test_token.py b/tests/test_token.py index e99ebe001..09cdf5fe5 100644 --- a/tests/test_token.py +++ b/tests/test_token.py @@ -240,7 +240,7 @@ def test_fully_authenticated_encrypted_token_protected_header(client): # deserialized it. Jose header is visible in plaintext. assert "exp" in token.token.jose_header expiration_time = datetime.datetime.fromisoformat(token.token.jose_header["exp"]) - token_issue = expiration_time - datetime.timedelta(hours=48) + token_issue = expiration_time - datetime.timedelta(hours=168) assert before_token_issue < token_issue assert after_token_issue > token_issue From 6f2aab778a68716d5d2ad1b704b8d1b5de646922 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Wed, 16 Mar 2022 09:44:25 +0100 Subject: [PATCH 103/293] Update dds_web/api/user.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Linus Östberg --- dds_web/api/user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index 6674581c8..c6dc10fba 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -938,7 +938,7 @@ class UnitUsers(flask_restful.Resource): @auth.login_required(role=["Super Admin", "Unit Admin", "Unit Personnel"]) @logging_bind_request def get(self): - """Get and return unit users within the unit the current user is connected to.""" + """List unit users within the unit the current user is connected to, or the one defined by a superadmin.""" unit_users = {} if not auth.current_user().is_active: From 1a93c4ff967299a3dec284945dbe42b8b258d493 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Wed, 16 Mar 2022 09:48:23 +0100 Subject: [PATCH 104/293] change to linus suggestion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Linus Östberg --- dds_web/api/user.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index c6dc10fba..5423f5fe3 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -951,11 +951,7 @@ def get(self): if auth.current_user().role == "Super Admin": json_input = flask.request.json - if not json_input: - raise ddserr.DDSArgumentError(message="Unit public id missing.") - - unit = json_input.get("unit") - if not unit: + if not json_input or not (unit := json_input.get("unit")): raise ddserr.DDSArgumentError(message="Unit public id missing.") unit_row = models.Unit.query.filter_by(public_id=unit).one_or_none() From 725a5514a3be1804db366a27297c73b5bcb72488 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Wed, 16 Mar 2022 10:04:23 +0100 Subject: [PATCH 105/293] Update dds_web/api/user.py --- dds_web/api/user.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index 5423f5fe3..c6dc10fba 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -951,7 +951,11 @@ def get(self): if auth.current_user().role == "Super Admin": json_input = flask.request.json - if not json_input or not (unit := json_input.get("unit")): + if not json_input: + raise ddserr.DDSArgumentError(message="Unit public id missing.") + + unit = json_input.get("unit") + if not unit: raise ddserr.DDSArgumentError(message="Unit public id missing.") unit_row = models.Unit.query.filter_by(public_id=unit).one_or_none() From b29bbf441cb4969fd62d275a11454c277cf14c7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Wed, 16 Mar 2022 11:55:53 +0100 Subject: [PATCH 106/293] project statuses refactored --- dds_web/api/project.py | 213 ++++++++++++++++++++++++++++------------- 1 file changed, 144 insertions(+), 69 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index ef8bd67a6..50f06632f 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -6,6 +6,9 @@ # Standard Library import http +from os import abort +from pyexpat import model +from dds_web import database # Installed import flask_restful @@ -94,87 +97,44 @@ def post(self): ]: raise DDSArgumentError("Invalid status") + # Check if valid status transition + if not self.is_transition_possible(project.current_status, new_status): + raise DDSArgumentError("Invalid status transition") + + # Override default to send email send_email = json_input.get("send_email", True) + # Initial variable definition curr_date = dds_web.utils.current_time() is_aborted = False - add_deadline = None - - if not self.is_transition_possible(project.current_status, new_status): - raise DDSArgumentError("Invalid status transition") # Moving to Available if new_status == "Available": - # Optional int arg deadline in days - deadline = json_input.get("deadline", project.responsible_unit.days_in_available) - add_deadline = dds_web.utils.current_time(to_midnight=True) + datetime.timedelta( - days=deadline + deadline_in = json_input.get("deadline", project.responsible_unit.days_in_available) + new_status_row = self.release_project( + project=project, current_time=curr_date, deadline_in=deadline_in ) - if project.current_status == "Expired": - # Project can only move from Expired 2 times - if project.times_expired > 2: - raise DDSArgumentError( - "Project availability limit: Project cannot be made Available any more times" - ) - else: # current status is in progress - if project.has_been_available: - # No change in deadline if made available before - add_deadline = project.current_deadline - else: - project.released = curr_date - - # Moving to Expired - if new_status == "Expired": - deadline = json_input.get("deadline", project.responsible_unit.days_in_expired) - add_deadline = dds_web.utils.current_time(to_midnight=True) + datetime.timedelta( - days=deadline + elif new_status == "In Progress": + new_status_row = self.retract_project(project=project, current_time=curr_date) + elif new_status == "Expired": + deadline_in = json_input.get("deadline", project.responsible_unit.days_in_expired) + new_status_row = self.expire_project( + project=project, current_time=curr_date, deadline_in=deadline_in ) - - # Moving to Deleted - if new_status == "Deleted": - # Can only be Deleted if never made Available - if project.has_been_available: - raise DDSArgumentError( - "Project cannot be deleted if it has ever been made available, abort it instead" - ) - project.is_active = False - - # Moving to Archived - if new_status == "Archived": + elif new_status == "Deleted": + new_status_row, delete_message = self.delete_project( + project=project, current_time=curr_date + ) + elif new_status == "Archived": is_aborted = json_input.get("is_aborted", False) - if project.current_status == "In Progress": - if not (project.has_been_available and is_aborted): - raise DDSArgumentError( - "Project cannot be archived from this status but can be aborted if it has ever been made available" - ) - project.is_active = False + new_status_row, delete_message = self.archive_project( + project=project, current_time=curr_date, aborted=is_aborted + ) - add_status = models.ProjectStatuses( - **{"project_id": project.id, "status": new_status, "date_created": curr_date}, - deadline=add_deadline, - is_aborted=is_aborted, - ) - delete_message = "" try: - project.project_statuses.append(add_status) - if not project.is_active: - # Deletes files (also commits session in the function - possibly refactor later) - RemoveContents().delete_project_contents(project=project) - delete_message = f"\nAll files in {project.public_id} deleted" - if new_status in ["Deleted", "Archived"]: - self.rm_project_user_keys(project=project) - if new_status == "Deleted" or is_aborted: - # Delete metadata from project row - project = self.delete_project_info(project) - delete_message += " and project info cleared" + project.project_statuses.append(new_status_row) db.session.commit() - except ( - sqlalchemy.exc.SQLAlchemyError, - TypeError, - DatabaseError, - DeletionError, - BucketNotFoundError, - ) as err: + except (sqlalchemy.exc.SQLAlchemyError) as err: flask.current_app.logger.exception(err) db.session.rollback() raise DatabaseError(message="Server Error: Status was not updated") from err @@ -211,6 +171,122 @@ def is_transition_possible(self, current_status, new_status): break return result + def release_project( + self, project: models.Project, current_time: datetime.datetime, deadline_in: int + ) -> models.ProjectStatuses: + """Release project: Make status Available. + + Only allowed from In Progress and Expired. + """ + if deadline_in > 90: + raise DDSArgumentError( + message="The deadline needs to be less than (or equal to) 90 days." + ) + + deadline = dds_web.utils.current_time(to_midnight=True) + datetime.timedelta( + days=deadline_in + ) + + # Project can only move from Expired 2 times + if project.current_status == "Expired": + if project.times_expired > 2: + raise DDSArgumentError( + "Project availability limit: Project cannot be made Available any more times" + ) + else: # current status is in progress + if project.has_been_available: + # No change in deadline if made available before + deadline = project.current_deadline + else: + project.released = current_time + + # Create row in ProjectStatuses + return models.ProjectStatuses( + status="Available", date_created=current_time, deadline=deadline + ) + + def retract_project(project: models.Project, current_time: datetime.datetime): + """Retract project: Make status In Progress. + + Only possible from Available. + """ + + return models.ProjectStatuses(status="In Progress", date_created=current_time) + + def expire_project( + self, project: models.Project, current_time: datetime.datetime, deadline_in: int + ) -> models.ProjectStatuses: + """Expire project: Make status Expired. + + Only possible Available. + """ + if deadline_in > 30: + raise DDSArgumentError( + message="The deadline needs to be less than (or equal to) 30 days." + ) + + deadline = dds_web.utils.current_time(to_midnight=True) + datetime.timedelta( + days=deadline_in + ) + return models.ProjectStatuses( + status="Expired", date_created=current_time, deadline=deadline + ) + + def delete_project(self, project: models.Project, current_time: datetime.datetime): + """Delete project: Make status Deleted. + + Only possible from In Progress. + """ + # Can only be Deleted if never made Available + if project.has_been_available: + raise DDSArgumentError( + "Project cannot be deleted if it has ever been made available, abort it instead." + ) + project.is_active = False + + # Deletes files (also commits session in the function - possibly refactor later) + RemoveContents().delete_project_contents(project=project) + self.rm_project_user_keys(project=project) + + # Delete metadata from project row + self.delete_project_info(proj=project) + delete_message = ( + f"\nAll files in project '{project.public_id}' deleted and project info cleared." + ) + + return models.ProjectStatuses(status="Deleted", date_created=current_time), delete_message + + def archive_project( + self, project: models.Project, current_time: datetime.datetime, aborted: bool = False + ): + """Archive project: Make status Archived. + + Only possible from Available and Expired. + """ + if project.current_status == "In Progress": + if not (project.has_been_available and aborted): + raise DDSArgumentError( + "Project cannot be archived from this status but can be aborted if it has ever been made available" + ) + project.is_active = False + + # Deletes files (also commits session in the function - possibly refactor later) + RemoveContents().delete_project_contents(project=project) + delete_message = f"\nAll files in {project.public_id} deleted" + self.rm_project_user_keys(project=project) + + # Delete metadata from project row + if aborted: + project = self.delete_project_info(project) + delete_message += " and project info cleared" + + return ( + models.ProjectStatuses( + status="Archived", date_created=current_time, is_aborted=aborted + ), + delete_message, + ) + def rm_project_user_keys(self, project): """Remove ProjectUserKey rows for specified project.""" for project_key in project.project_user_keys: @@ -228,7 +304,6 @@ def delete_project_info(self, proj): # Delete User associations for user in proj.researchusers: db.session.delete(user) - return proj class GetPublic(flask_restful.Resource): From 492908c9e2b7c61b247713dd160a8551a2192b42 Mon Sep 17 00:00:00 2001 From: Zishan Mirza Date: Wed, 16 Mar 2022 11:11:45 +0000 Subject: [PATCH 107/293] Update text This patch updates the text. Signed-off-by: Zishan Mirza --- dds_web/api/user.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index b1bcdafe8..294028527 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -383,14 +383,14 @@ def add_to_project(whom, project, role, send_email=True): AddUser.compose_and_send_email_to_user(whom, "project_release", project=project) flask.current_app.logger.debug( - f"{str(whom)} was associated with {str(project)} as Owner={is_owner}." + f"{str(whom)} was associated to the {str(project)} as a Researcher={is_owner}." ) return { "status": http.HTTPStatus.OK, "message": ( - f"{str(whom)} was associated with " - f"{str(project)} as Owner={is_owner}. An e-mail notification has{' not ' if not send_email else ' '}been sent." + f"{str(whom)} was associated to the " + f"{str(project)} as a Researcher={is_owner}. An e-mail notification has{' not ' if not send_email else ' '}been sent." ), } From ac6e3f0055b3a169c6e781c8e060ba0509304e42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Wed, 16 Mar 2022 13:20:27 +0100 Subject: [PATCH 108/293] Update tests/test_unit_list.py --- tests/test_unit_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_unit_list.py b/tests/test_unit_list.py index 79a62b6d4..14694f86b 100644 --- a/tests/test_unit_list.py +++ b/tests/test_unit_list.py @@ -25,7 +25,7 @@ def get_token(username, client): return tests.UserAuth(tests.USER_CREDENTIALS[username]).token(client) -def list_units_as_not_superadmin(client): +def test_list_units_as_not_superadmin(client): """Only Super Admin can list users.""" no_access_users = users.copy() no_access_users.pop("Super Admin") From 9073421473d3e24dcee278a88e9941a9f84824a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Wed, 16 Mar 2022 13:20:41 +0100 Subject: [PATCH 109/293] Update tests/test_unit_list.py --- tests/test_unit_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_unit_list.py b/tests/test_unit_list.py index 14694f86b..bfeb2c7ff 100644 --- a/tests/test_unit_list.py +++ b/tests/test_unit_list.py @@ -36,7 +36,7 @@ def test_list_units_as_not_superadmin(client): assert response.status_code == http.HTTPStatus.UNAUTHORIZED -def list_units_as_super_admin(client): +def test_list_units_as_super_admin(client): """List units as Super Admin.""" all_units = models.Unit.query.all() From a8f31fdc9ddfaa23a1609efd4c9c2200f311ba19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Wed, 16 Mar 2022 13:31:03 +0100 Subject: [PATCH 110/293] unused active check --- dds_web/api/user.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index d4eb8799c..2ee892eba 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -943,14 +943,6 @@ def get(self): """List unit users within the unit the current user is connected to, or the one defined by a superadmin.""" unit_users = {} - if not auth.current_user().is_active: - raise ddserr.AccessDeniedError( - message=( - "Your account has been deactivated. " - "You cannot list the users within your unit." - ) - ) - if auth.current_user().role == "Super Admin": json_input = flask.request.json if not json_input: From 35aaff3708932bd30501e34fe31bb35165a1144c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Wed, 16 Mar 2022 13:37:27 +0100 Subject: [PATCH 111/293] raise access denied if user is inactive --- dds_web/security/auth.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dds_web/security/auth.py b/dds_web/security/auth.py index 84c3cda93..fe2aac33f 100644 --- a/dds_web/security/auth.py +++ b/dds_web/security/auth.py @@ -239,7 +239,11 @@ def __user_from_subject(subject): """Get user row from username.""" if subject: user = models.User.query.get(subject) - if user and user.is_active: + if user: + if not user.is_active: + raise AccessDeniedError( + message=("Your account has been deactivated. You cannot use the DDS.") + ) return user From 36d7396a3e5083a5dec13508dc6fbc2518d23a83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Wed, 16 Mar 2022 13:43:15 +0100 Subject: [PATCH 112/293] test fixed --- tests/test_unit_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_unit_list.py b/tests/test_unit_list.py index bfeb2c7ff..5b4666f37 100644 --- a/tests/test_unit_list.py +++ b/tests/test_unit_list.py @@ -33,7 +33,7 @@ def test_list_units_as_not_superadmin(client): for u in no_access_users: token = get_token(username=users[u], client=client) response = client.get(tests.DDSEndpoint.LIST_UNITS_ALL, headers=token) - assert response.status_code == http.HTTPStatus.UNAUTHORIZED + assert response.status_code == http.HTTPStatus.FORBIDDEN def test_list_units_as_super_admin(client): From e8c18b3cb14d7b16481837376fe663a9202288b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Wed, 16 Mar 2022 13:45:13 +0100 Subject: [PATCH 113/293] fixed another test --- tests/test_users_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_users_list.py b/tests/test_users_list.py index bb608e7b3..129cf2796 100644 --- a/tests/test_users_list.py +++ b/tests/test_users_list.py @@ -47,7 +47,7 @@ def test_list_unitusers_with_unit_personnel_and_admin_deactivated(client): response = client.get(tests.DDSEndpoint.LIST_UNIT_USERS, headers=token) # Unauth and not forbidden because the user object is not returned from the token - assert response.status_code == http.HTTPStatus.UNAUTHORIZED + assert response.status_code == http.HTTPStatus.FORBIDDEN def test_list_unitusers_with_unit_personnel_and_admin_ok(client): From 49e701784d0a6b5738480f2fc1661389a251d83d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Wed, 16 Mar 2022 13:58:06 +0100 Subject: [PATCH 114/293] incorrect comment --- tests/test_users_list.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_users_list.py b/tests/test_users_list.py index 129cf2796..dc351fe7c 100644 --- a/tests/test_users_list.py +++ b/tests/test_users_list.py @@ -45,8 +45,6 @@ def test_list_unitusers_with_unit_personnel_and_admin_deactivated(client): # Try to list users - should only work if active - not now response = client.get(tests.DDSEndpoint.LIST_UNIT_USERS, headers=token) - - # Unauth and not forbidden because the user object is not returned from the token assert response.status_code == http.HTTPStatus.FORBIDDEN From 28cda21b0dc42d73954b8552a86562aecc676db4 Mon Sep 17 00:00:00 2001 From: Zishan Mirza Date: Wed, 16 Mar 2022 13:18:43 +0000 Subject: [PATCH 115/293] Update text Updated unit tests. Signed-off-by: Zishan Mirza --- tests/test_project_creation.py | 2 +- tests/test_project_status.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_project_creation.py b/tests/test_project_creation.py index 4d05d951a..5f00603a8 100644 --- a/tests/test_project_creation.py +++ b/tests/test_project_creation.py @@ -421,7 +421,7 @@ def test_create_project_with_users(client, boto3_session): assert response.status_code == http.HTTPStatus.OK assert response.json and response.json.get("user_addition_statuses") for x in response.json.get("user_addition_statuses"): - assert "associated with Project" in x + assert "associated to the Project" in x resp_json = response.json created_proj = models.Project.query.filter_by(public_id=resp_json["project_id"]).one_or_none() diff --git a/tests/test_project_status.py b/tests/test_project_status.py index 0d24eb722..df0f00c6c 100644 --- a/tests/test_project_status.py +++ b/tests/test_project_status.py @@ -379,7 +379,7 @@ def test_set_project_to_available_no_mail(module_client, boto3_session): assert response.status_code == http.HTTPStatus.OK assert response.json and response.json.get("user_addition_statuses") for x in response.json.get("user_addition_statuses"): - assert "associated with Project" in x + assert "associated to the Project" in x public_project_id = response.json.get("project_id") From 625b60cf871ecc423a13ed577129076d116753a1 Mon Sep 17 00:00:00 2001 From: Zishan Mirza Date: Wed, 16 Mar 2022 14:54:26 +0000 Subject: [PATCH 116/293] Update text Updated the text and unit tests. Signed-off-by: Zishan Mirza --- dds_web/api/user.py | 6 +++--- tests/test_project_creation.py | 2 +- tests/test_project_status.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index 69b3c429a..9fd70bcb6 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -383,14 +383,14 @@ def add_to_project(whom, project, role, send_email=True): AddUser.compose_and_send_email_to_user(whom, "project_release", project=project) flask.current_app.logger.debug( - f"{str(whom)} was associated to the {str(project)} as a Researcher={is_owner}." + f"{str(whom)} was given access to the {str(project)} as a {'Product Owner' if is_owner else 'Researcher'}." ) return { "status": http.HTTPStatus.OK, "message": ( - f"{str(whom)} was associated to the " - f"{str(project)} as a Researcher={is_owner}. An e-mail notification has{' not ' if not send_email else ' '}been sent." + f"{str(whom)} was given access to the " + f"{str(project)} as a {'Product Owner' if is_owner else 'Researcher'}. An e-mail notification has{' not ' if not send_email else ' '}been sent." ), } diff --git a/tests/test_project_creation.py b/tests/test_project_creation.py index 5f00603a8..4ec1900fe 100644 --- a/tests/test_project_creation.py +++ b/tests/test_project_creation.py @@ -421,7 +421,7 @@ def test_create_project_with_users(client, boto3_session): assert response.status_code == http.HTTPStatus.OK assert response.json and response.json.get("user_addition_statuses") for x in response.json.get("user_addition_statuses"): - assert "associated to the Project" in x + assert "given access to the Project" in x resp_json = response.json created_proj = models.Project.query.filter_by(public_id=resp_json["project_id"]).one_or_none() diff --git a/tests/test_project_status.py b/tests/test_project_status.py index df0f00c6c..685d910ff 100644 --- a/tests/test_project_status.py +++ b/tests/test_project_status.py @@ -379,7 +379,7 @@ def test_set_project_to_available_no_mail(module_client, boto3_session): assert response.status_code == http.HTTPStatus.OK assert response.json and response.json.get("user_addition_statuses") for x in response.json.get("user_addition_statuses"): - assert "associated to the Project" in x + assert "given access to the Project" in x public_project_id = response.json.get("project_id") From 02f0b459e40a195f7a27bf9a3335c5351c32da48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Wed, 16 Mar 2022 16:51:04 +0100 Subject: [PATCH 117/293] fixed tests according to changes --- dds_web/api/project.py | 88 ++++++++++++++++++++++++------------ tests/test_project_status.py | 29 ++++++++---- 2 files changed, 77 insertions(+), 40 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 50f06632f..9faf76f28 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -88,18 +88,8 @@ def post(self): # Check if valid status json_input = flask.request.json new_status = json_input.get("new_status") - if new_status not in [ - "In Progress", - "Deleted", - "Available", - "Expired", - "Archived", - ]: - raise DDSArgumentError("Invalid status") - - # Check if valid status transition - if not self.is_transition_possible(project.current_status, new_status): - raise DDSArgumentError("Invalid status transition") + if not new_status: + raise DDSArgumentError(message="No status transition provided. Specify the new status.") # Override default to send email send_email = json_input.get("send_email", True) @@ -107,6 +97,7 @@ def post(self): # Initial variable definition curr_date = dds_web.utils.current_time() is_aborted = False + delete_message = "" # Moving to Available if new_status == "Available": @@ -130,6 +121,8 @@ def post(self): new_status_row, delete_message = self.archive_project( project=project, current_time=curr_date, aborted=is_aborted ) + else: + raise DDSArgumentError(message="Invalid status") try: project.project_statuses.append(new_status_row) @@ -156,20 +149,37 @@ def post(self): ) return {"message": return_message} - def is_transition_possible(self, current_status, new_status): - """Check if the transition is valid""" - possible_transitions = [ - ("In Progress", ["Available", "Deleted", "Archived"]), - ("Available", ["In Progress", "Expired", "Archived"]), - ("Expired", ["Available", "Archived"]), - ] - result = False + def check_transition_possible(self, current_status, new_status): + """Check if the transition is valid.""" + valid_statuses = { + "In Progress": "retract", + "Available": "release", + "Deleted": "delete", + "Expired": "expire", + "Archived": "archive", + } + if new_status not in valid_statuses: + raise DDSArgumentError("Invalid status") - for transition in possible_transitions: - if current_status == transition[0] and new_status in transition[1]: - result = True - break - return result + possible_transitions = { + "In Progress": ["Available", "Deleted", "Archived"], + "Available": ["In Progress", "Expired", "Archived"], + "Expired": ["Available", "Archived"], + } + + current_transition = possible_transitions.get(current_status) + if not current_transition: + raise DDSArgumentError( + message=f"Cannot change status for a project that has the status '{current_status}'." + ) + + if new_status not in current_transition: + raise DDSArgumentError( + message=( + f"You cannot {valid_statuses[new_status]} a " + f"project that has the current status '{current_status}'." + ) + ) def release_project( self, project: models.Project, current_time: datetime.datetime, deadline_in: int @@ -178,6 +188,11 @@ def release_project( Only allowed from In Progress and Expired. """ + # Check if valid status transition + self.check_transition_possible( + current_status=project.current_status, new_status="Available" + ) + if deadline_in > 90: raise DDSArgumentError( message="The deadline needs to be less than (or equal to) 90 days." @@ -205,11 +220,15 @@ def release_project( status="Available", date_created=current_time, deadline=deadline ) - def retract_project(project: models.Project, current_time: datetime.datetime): + def retract_project(self, project: models.Project, current_time: datetime.datetime): """Retract project: Make status In Progress. Only possible from Available. """ + # Check if valid status transition + self.check_transition_possible( + current_status=project.current_status, new_status="In Progress" + ) return models.ProjectStatuses(status="In Progress", date_created=current_time) @@ -220,6 +239,9 @@ def expire_project( Only possible Available. """ + # Check if valid status transition + self.check_transition_possible(current_status=project.current_status, new_status="Expired") + if deadline_in > 30: raise DDSArgumentError( message="The deadline needs to be less than (or equal to) 30 days." @@ -237,6 +259,9 @@ def delete_project(self, project: models.Project, current_time: datetime.datetim Only possible from In Progress. """ + # Check if valid status transition + self.check_transition_possible(current_status=project.current_status, new_status="Deleted") + # Can only be Deleted if never made Available if project.has_been_available: raise DDSArgumentError( @@ -251,7 +276,7 @@ def delete_project(self, project: models.Project, current_time: datetime.datetim # Delete metadata from project row self.delete_project_info(proj=project) delete_message = ( - f"\nAll files in project '{project.public_id}' deleted and project info cleared." + f"\nAll files in project '{project.public_id}' deleted and project info cleared" ) return models.ProjectStatuses(status="Deleted", date_created=current_time), delete_message @@ -263,10 +288,14 @@ def archive_project( Only possible from Available and Expired. """ - if project.current_status == "In Progress": + # Check if valid status transition + self.check_transition_possible(current_status=project.current_status, new_status="Archived") + + if not aborted and project.current_status == "In Progress": if not (project.has_been_available and aborted): raise DDSArgumentError( - "Project cannot be archived from this status but can be aborted if it has ever been made available" + f"You cannot archive a project that has been made available previously. " + "Abort the project if you wish to proceed." ) project.is_active = False @@ -300,7 +329,6 @@ def delete_project_info(self, proj): proj.description = None proj.pi = None proj.public_key = None - proj.created_by = None # Delete User associations for user in proj.researchusers: db.session.delete(user) diff --git a/tests/test_project_status.py b/tests/test_project_status.py index 0d24eb722..fbb7bd4fe 100644 --- a/tests/test_project_status.py +++ b/tests/test_project_status.py @@ -33,7 +33,7 @@ "pi", "public_key", # "unit_id", - "created_by", + # "created_by", # "is_active", # "date_updated", ] @@ -173,7 +173,7 @@ def test_aborted_project(module_client, boto3_session): assert response.status_code == http.HTTPStatus.BAD_REQUEST assert project.current_status == "In Progress" assert ( - "Project cannot be archived from this status but can be aborted if it has ever been made available" + "You cannot archive a project that has been made available previously" in response.json["message"] ) @@ -320,7 +320,10 @@ def test_check_invalid_transitions_from_in_progress(module_client, test_project) assert response.status_code == http.HTTPStatus.BAD_REQUEST assert project.current_status == "In Progress" - assert "Invalid status transition" in response.json["message"] + assert ( + "You cannot expire a project that has the current status 'In Progress'." + in response.json["message"] + ) # In Progress to Archived new_status["new_status"] = "Archived" @@ -334,7 +337,7 @@ def test_check_invalid_transitions_from_in_progress(module_client, test_project) assert response.status_code == http.HTTPStatus.BAD_REQUEST assert project.current_status == "In Progress" assert ( - "Project cannot be archived from this status but can be aborted if it has ever been made available" + "You cannot archive a project that has been made available previously" in response.json["message"] ) @@ -584,7 +587,10 @@ def test_invalid_transitions_from_expired(module_client, test_project): ) assert response.status_code == http.HTTPStatus.BAD_REQUEST assert project.current_status == "Expired" - assert "Invalid status transition" in response.json["message"] + assert ( + "You cannot retract a project that has the current status 'Expired'" + in response.json["message"] + ) # Expired to Deleted new_status["new_status"] = "Deleted" @@ -596,7 +602,10 @@ def test_invalid_transitions_from_expired(module_client, test_project): ) assert response.status_code == http.HTTPStatus.BAD_REQUEST assert project.current_status == "Expired" - assert "Invalid status transition" in response.json["message"] + assert ( + "You cannot delete a project that has the current status 'Expired'" + in response.json["message"] + ) def test_set_project_to_archived(module_client, test_project, boto3_session): @@ -639,7 +648,7 @@ def test_invalid_transitions_from_archived(module_client, test_project): ) assert response.status_code == http.HTTPStatus.BAD_REQUEST assert project.current_status == "Archived" - assert "Invalid status transition" in response.json["message"] + assert "Cannot change status for a project" in response.json["message"] # Archived to Deleted new_status["new_status"] = "Deleted" @@ -651,7 +660,7 @@ def test_invalid_transitions_from_archived(module_client, test_project): ) assert response.status_code == http.HTTPStatus.BAD_REQUEST assert project.current_status == "Archived" - assert "Invalid status transition" in response.json["message"] + assert "Cannot change status for a project" in response.json["message"] # Archived to Available new_status["new_status"] = "Available" @@ -663,7 +672,7 @@ def test_invalid_transitions_from_archived(module_client, test_project): ) assert response.status_code == http.HTTPStatus.BAD_REQUEST assert project.current_status == "Archived" - assert "Invalid status transition" in response.json["message"] + assert "Cannot change status for a project" in response.json["message"] # Archived to Expired new_status["new_status"] = "Expired" @@ -675,4 +684,4 @@ def test_invalid_transitions_from_archived(module_client, test_project): ) assert response.status_code == http.HTTPStatus.BAD_REQUEST assert project.current_status == "Archived" - assert "Invalid status transition" in response.json["message"] + assert "Cannot change status for a project" in response.json["message"] From df1ce78c60606dcf754e2fc0830863b397993dd4 Mon Sep 17 00:00:00 2001 From: Johannes Alneberg Date: Wed, 16 Mar 2022 16:59:01 +0100 Subject: [PATCH 118/293] Is None doesn't work with sqlalchemy --- .../versions/666003748d14_change_active_nullable_and_default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/versions/666003748d14_change_active_nullable_and_default.py b/migrations/versions/666003748d14_change_active_nullable_and_default.py index 486520499..7ea464f47 100644 --- a/migrations/versions/666003748d14_change_active_nullable_and_default.py +++ b/migrations/versions/666003748d14_change_active_nullable_and_default.py @@ -20,7 +20,7 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### user_table = sa.sql.table("users", sa.sql.column("active", mysql.TINYINT(display_width=1))) - op.execute(user_table.update().where(user_table.c.active is None).values(active=False)) + op.execute(user_table.update().where(user_table.c.active == None).values(active=False)) op.alter_column("users", "active", existing_type=mysql.TINYINT(display_width=1), nullable=False) # ### end Alembic commands ### From c99b60fd8c17cc5533dee4addc621ae85c000b99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Thu, 17 Mar 2022 11:32:47 +0100 Subject: [PATCH 119/293] add check for unit admins --- dds_web/api/project.py | 12 +++++++++++- dds_web/utils.py | 20 ++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index ef8bd67a6..991330b5a 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -441,8 +441,18 @@ class CreateProject(flask_restful.Resource): @handle_validation_errors def post(self): """Create a new project.""" - # Add a new project to db p_info = flask.request.json + + # Verify enough number of Unit Admins or return message + force_create = p_info.get("force", False) + warning_message = dds_web.utils.verify_enough_unit_admins( + unit_id=auth.current_user().unit.id, force_create=force_create + ) + if warning_message: + return {"warning": warning_message} + + # Add a new project to db + new_project = project_schemas.CreateProjectSchema().load(p_info) db.session.add(new_project) diff --git a/dds_web/utils.py b/dds_web/utils.py index 70b829682..1022c2f57 100644 --- a/dds_web/utils.py +++ b/dds_web/utils.py @@ -13,6 +13,7 @@ # Installed from contextlib import contextmanager import flask +from dds_web.errors import AccessDeniedError import flask_mail import flask_login @@ -181,6 +182,25 @@ def _email_taken(form, field): #################################################################################################### +def verify_enough_unit_admins(unit_id: str, force_create: bool = False): + """Verify that the unit has enough Unit Admins.""" + num_admins = models.UnitUser.query.filter_by(is_admin=True, unit_id=unit_id).count() + if num_admins < 2: + raise AccessDeniedError( + message=( + "Your unit does not have enough Unit Admins. " + "At least two Unit Admins are required for a project to be created." + ) + ) + + if num_admins < 3 and not force_create: + return ( + f"Your unit only has {num_admins} Unit Admins. This poses a high risk of data loss. " + "We HIGHLY recommend that you do not create this project until there are more Unit " + "Admins connected to your unit." + ) + + def valid_chars_in_username(indata): """Check if the username contains only valid characters.""" return bool(re.search(r"^[a-zA-Z0-9_\.-]+$", indata)) From 76285bb51503f645092079a4bb74144eafae9fec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Thu, 17 Mar 2022 12:51:14 +0100 Subject: [PATCH 120/293] fix tests and add new for checking num admins --- dds_web/api/project.py | 4 + tests/test_project_creation.py | 190 +++++++++++++++++++++++++++++++++ 2 files changed, 194 insertions(+) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 991330b5a..54500e940 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -5,6 +5,7 @@ #################################################################################################### # Standard Library +from email import message import http # Installed @@ -445,6 +446,9 @@ def post(self): # Verify enough number of Unit Admins or return message force_create = p_info.get("force", False) + if not isinstance(force_create, bool): + raise DDSArgumentError(message="`force` is a boolean value: True or False.") + warning_message = dds_web.utils.verify_enough_unit_admins( unit_id=auth.current_user().unit.id, force_create=force_create ) diff --git a/tests/test_project_creation.py b/tests/test_project_creation.py index 4ec1900fe..90671d6fa 100644 --- a/tests/test_project_creation.py +++ b/tests/test_project_creation.py @@ -4,6 +4,8 @@ import http import datetime import json +from tokenize import Triple +from turtle import update import unittest import time @@ -42,9 +44,86 @@ ], } + +def create_unit_admins(num_admins, unit_id=1): + new_admins = [] + for i in range(1, num_admins + 1): + new_admins.append( + models.UnitUser( + **{ + "username": "unit_admin_" + str(i), + "name": "Unit Admin " + str(i), + "password": "password", + "is_admin": True, + "unit_id": unit_id, + } + ) + ) + + db.session.add_all(new_admins) + db.session.commit() + + # TESTS #################################################################################### TESTS # +def test_create_project_too_few_unit_admins(client): + """There needs to be at least 2 Unit Admins.""" + response = client.post( + tests.DDSEndpoint.PROJECT_CREATE, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), + json=proj_data, + ) + assert response.status_code == http.HTTPStatus.FORBIDDEN + response_json = response.json + assert response_json + assert "Your unit does not have enough Unit Admins" in response_json.get("message") + + +def test_create_project_two_unit_admins(client): + """There needs to be at least 2 Unit Admins.""" + create_unit_admins(num_admins=1) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 2 + + response = client.post( + tests.DDSEndpoint.PROJECT_CREATE, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), + json=proj_data, + ) + assert response.status_code == http.HTTPStatus.OK + response_json = response.json + assert response_json + assert "Your unit only has 2 Unit Admins" in response_json.get("warning") + + +def test_create_project_two_unit_admins_force(client): + """The force option (not in cli) can be used to create a project even if there are + less than 3 Unit Admins.""" + create_unit_admins(num_admins=1) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 2 + + # Use force + updated_proj_data = proj_data.copy() + updated_proj_data["force"] = True + response = client.post( + tests.DDSEndpoint.PROJECT_CREATE, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), + json=updated_proj_data, + ) + assert response.status_code == http.HTTPStatus.OK + created_proj = models.Project.query.filter_by( + created_by="unitadmin", + title=updated_proj_data["title"], + pi=updated_proj_data["pi"], + description=updated_proj_data["description"], + ).one_or_none() + assert created_proj + + def test_create_project_empty(client): """Make empty request.""" response = client.post( @@ -59,6 +138,13 @@ def test_create_project_empty(client): def test_create_project_unknown_field(client): """Make request with unknown field passed.""" + # Make sure there's 3 unit admins for unit + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + + # Attempt creating project response = client.post( tests.DDSEndpoint.PROJECT_CREATE, headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), @@ -75,6 +161,11 @@ def test_create_project_unknown_field(client): def test_create_project_missing_title(client): """Make request with missing title.""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + proj_data_no_title = proj_data.copy() proj_data_no_title.pop("title") @@ -92,6 +183,11 @@ def test_create_project_missing_title(client): def test_create_project_none_title(client): """Make request with missing title.""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + proj_data_none_title = proj_data.copy() proj_data_none_title["title"] = None @@ -109,6 +205,11 @@ def test_create_project_none_title(client): def test_create_project_no_description(client): """Make request with missing title.""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + proj_data_no_description = proj_data.copy() proj_data_no_description.pop("description") @@ -127,6 +228,11 @@ def test_create_project_no_description(client): def test_create_project_none_description(client): """Make request with missing title.""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + proj_data_none_description = proj_data.copy() proj_data_none_description["description"] = None @@ -145,6 +251,11 @@ def test_create_project_none_description(client): def test_create_project_no_pi(client): """Make request with missing title.""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + proj_data_no_pi = proj_data.copy() proj_data_no_pi.pop("pi") @@ -163,6 +274,11 @@ def test_create_project_no_pi(client): def test_create_project_none_pi(client): """Make request with missing title.""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + proj_data_none_pi = proj_data.copy() proj_data_none_pi["pi"] = None @@ -181,6 +297,11 @@ def test_create_project_none_pi(client): def test_create_project_without_credentials(client): """Create project without valid user credentials.""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + response = client.post( tests.DDSEndpoint.PROJECT_CREATE, headers=tests.UserAuth(tests.USER_CREDENTIALS["researchuser"]).token(client), @@ -198,6 +319,11 @@ def test_create_project_without_credentials(client): def test_create_project_with_credentials(client, boto3_session): """Create project with correct credentials.""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + time_before_run = datetime.datetime.utcnow() time.sleep(1) response = client.post( @@ -221,6 +347,11 @@ def test_create_project_with_credentials(client, boto3_session): def test_create_project_no_title(client): """Create project without a title specified.""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + response = client.post( tests.DDSEndpoint.PROJECT_CREATE, headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), @@ -237,6 +368,11 @@ def test_create_project_no_title(client): def test_create_project_title_too_short(client): """Create a project with too short title.""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + proj_data_short_title = proj_data.copy() proj_data_short_title["title"] = "" response = client.post( @@ -257,6 +393,11 @@ def test_create_project_title_too_short(client): def test_create_project_with_malformed_json(client): """Create a project with malformed project info.""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + response = client.post( tests.DDSEndpoint.PROJECT_CREATE, headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), @@ -274,6 +415,11 @@ def test_create_project_with_malformed_json(client): def test_create_project_sensitive(client, boto3_session): """Create a sensitive project.""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + p_data = proj_data p_data["non_sensitive"] = False response = client.post( @@ -293,6 +439,11 @@ def test_create_project_sensitive(client, boto3_session): def test_create_project_description_too_short(client): """Create a project with too short description.""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + proj_data_short_description = proj_data.copy() proj_data_short_description["description"] = "" response = client.post( @@ -313,6 +464,11 @@ def test_create_project_description_too_short(client): def test_create_project_pi_too_short(client): """Create a project with too short PI.""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + proj_data_short_pi = proj_data.copy() proj_data_short_pi["pi"] = "" response = client.post( @@ -333,6 +489,11 @@ def test_create_project_pi_too_short(client): def test_create_project_pi_too_long(client): """Create a project with too long PI.""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + proj_data_long_pi = proj_data.copy() proj_data_long_pi["pi"] = "pi" * 128 response = client.post( @@ -353,6 +514,11 @@ def test_create_project_pi_too_long(client): def test_create_project_wrong_status(client, boto3_session): """Create a project with own status, should be overridden.""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + proj_data_wrong_status = proj_data.copy() proj_data_wrong_status["status"] = "Incorrect Status" response = client.post( @@ -373,6 +539,11 @@ def test_create_project_wrong_status(client, boto3_session): def test_create_project_sensitive_not_boolean(client): """Create project with incorrect non_sensitive format.""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + proj_data_sensitive_not_boolean = proj_data.copy() proj_data_sensitive_not_boolean["non_sensitive"] = "test" response = client.post( @@ -393,6 +564,11 @@ def test_create_project_sensitive_not_boolean(client): def test_create_project_date_created_overridden(client, boto3_session): """Create project with own date_created, should be overridden.""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + proj_data_date_created_own = proj_data.copy() proj_data_date_created_own["date_created"] = "test" response = client.post( @@ -413,6 +589,11 @@ def test_create_project_date_created_overridden(client, boto3_session): def test_create_project_with_users(client, boto3_session): """Create project and add users to the project.""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + response = client.post( tests.DDSEndpoint.PROJECT_CREATE, headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), @@ -448,6 +629,10 @@ def test_create_project_with_users(client, boto3_session): def test_create_project_with_invited_users(client, boto3_session): """Create project and invite users to the project.""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 response = client.post( tests.DDSEndpoint.PROJECT_CREATE, @@ -462,6 +647,11 @@ def test_create_project_with_invited_users(client, boto3_session): def test_create_project_with_unsuitable_roles(client, boto3_session): """Create project and add users with unsuitable roles to the project.""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + response = client.post( tests.DDSEndpoint.PROJECT_CREATE, headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), From 813653760e0432e26c0da202b1519aff097d3fc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Thu, 17 Mar 2022 12:52:10 +0100 Subject: [PATCH 121/293] fix tests and add new for checking num admins --- tests/test_project_creation.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/test_project_creation.py b/tests/test_project_creation.py index 90671d6fa..0b8b69ff4 100644 --- a/tests/test_project_creation.py +++ b/tests/test_project_creation.py @@ -124,6 +124,32 @@ def test_create_project_two_unit_admins_force(client): assert created_proj +def test_create_project_two_unit_admins_force(client): + """The force option (not in cli) can be used to create a project even if there are + less than 3 Unit Admins.""" + create_unit_admins(num_admins=1) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 2 + + # Use force + updated_proj_data = proj_data.copy() + updated_proj_data["force"] = "not correct" + response = client.post( + tests.DDSEndpoint.PROJECT_CREATE, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), + json=updated_proj_data, + ) + assert response.status_code == http.HTTPStatus.BAD_REQUEST + created_proj = models.Project.query.filter_by( + created_by="unitadmin", + title=updated_proj_data["title"], + pi=updated_proj_data["pi"], + description=updated_proj_data["description"], + ).one_or_none() + assert not created_proj + + def test_create_project_empty(client): """Make empty request.""" response = client.post( From f95635134cc0ca7262ebfa0f0442f095b86cc391 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Thu, 17 Mar 2022 12:58:39 +0100 Subject: [PATCH 122/293] final test solved --- dds_web/api/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 54500e940..d5d6870ae 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -445,7 +445,7 @@ def post(self): p_info = flask.request.json # Verify enough number of Unit Admins or return message - force_create = p_info.get("force", False) + force_create = p_info.pop("force", False) if not isinstance(force_create, bool): raise DDSArgumentError(message="`force` is a boolean value: True or False.") From 5b1841ff7aac5e9aa83f5cb386dcbcb47b9af94d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Thu, 17 Mar 2022 13:10:25 +0100 Subject: [PATCH 123/293] added create unit admins to other tests --- tests/test_project_status.py | 27 ++++++++++++++++++++++++++- tests/test_user_remove_association.py | 18 +++++++++++++++++- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/tests/test_project_status.py b/tests/test_project_status.py index 685d910ff..6c4709e0a 100644 --- a/tests/test_project_status.py +++ b/tests/test_project_status.py @@ -16,7 +16,8 @@ import dds_web import tests from tests.test_files_new import project_row, file_in_db, FIRST_NEW_FILE -from tests.test_project_creation import proj_data_with_existing_users +from tests.test_project_creation import proj_data_with_existing_users, create_unit_admins +from dds_web.database import models # CONFIG ################################################################################## CONFIG # @@ -42,6 +43,11 @@ @pytest.fixture(scope="module") def test_project(module_client): """Create a shared test project""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + with unittest.mock.patch.object(boto3.session.Session, "resource") as mock_session: response = module_client.post( tests.DDSEndpoint.PROJECT_CREATE, @@ -88,6 +94,10 @@ def test_submit_request_with_invalid_args(module_client, test_project): def test_set_project_to_deleted_from_in_progress(module_client, boto3_session): """Create project and set status to deleted""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 new_status = {"new_status": "Deleted"} response = module_client.post( @@ -142,6 +152,11 @@ def test_set_project_to_deleted_from_in_progress(module_client, boto3_session): def test_aborted_project(module_client, boto3_session): """Create a project and try to abort it""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + response = module_client.post( tests.DDSEndpoint.PROJECT_CREATE, headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(module_client), @@ -221,6 +236,11 @@ def test_aborted_project(module_client, boto3_session): def test_abort_from_in_progress_once_made_available(module_client, boto3_session): """Create project and abort it from In Progress after it has been made available""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + response = module_client.post( tests.DDSEndpoint.PROJECT_CREATE, headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(module_client), @@ -369,6 +389,11 @@ def test_set_project_to_available_valid_transition(module_client, test_project): def test_set_project_to_available_no_mail(module_client, boto3_session): """Set status to Available for test project, but skip sending mails""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + token = tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(module_client) response = module_client.post( diff --git a/tests/test_user_remove_association.py b/tests/test_user_remove_association.py index 49ba0a96d..30a75c6a6 100644 --- a/tests/test_user_remove_association.py +++ b/tests/test_user_remove_association.py @@ -5,12 +5,18 @@ # Own import tests -from tests.test_project_creation import proj_data_with_existing_users +from tests.test_project_creation import proj_data_with_existing_users, create_unit_admins +from dds_web.database import models def test_remove_user_from_project(client, boto3_session): """Remove an associated user from a project""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + response = client.post( tests.DDSEndpoint.PROJECT_CREATE, headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), @@ -36,6 +42,11 @@ def test_remove_user_from_project(client, boto3_session): def test_remove_not_associated_user_from_project(client, boto3_session): """Try to remove a user that exists in db but is not associated to a project""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + proj_data = copy.deepcopy(proj_data_with_existing_users) proj_data["users_to_add"].pop(1) @@ -63,6 +74,11 @@ def test_remove_not_associated_user_from_project(client, boto3_session): def test_remove_nonexistent_user_from_project(client, boto3_session): """Try to remove an nonexistent user from a project""" + create_unit_admins(num_admins=2) + + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + response = client.post( tests.DDSEndpoint.PROJECT_CREATE, headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client), From 3f0eb7be8741a74fdf3b3aa9b0c6d9a76b440dd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Thu, 17 Mar 2022 13:14:03 +0100 Subject: [PATCH 124/293] Update dds_web/api/project.py --- dds_web/api/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 9faf76f28..b51f039ce 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -291,7 +291,7 @@ def archive_project( # Check if valid status transition self.check_transition_possible(current_status=project.current_status, new_status="Archived") - if not aborted and project.current_status == "In Progress": + if project.current_status == "In Progress": if not (project.has_been_available and aborted): raise DDSArgumentError( f"You cannot archive a project that has been made available previously. " From a9c9215e7f3c331426d9c6387964a79d1b9f718a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Thu, 17 Mar 2022 13:51:56 +0100 Subject: [PATCH 125/293] change aborted check --- dds_web/api/project.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index b51f039ce..7c8339f4c 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -292,9 +292,9 @@ def archive_project( self.check_transition_possible(current_status=project.current_status, new_status="Archived") if project.current_status == "In Progress": - if not (project.has_been_available and aborted): + if project.has_been_available and not aborted: raise DDSArgumentError( - f"You cannot archive a project that has been made available previously. " + "You cannot archive a project that has been made available previously. " "Abort the project if you wish to proceed." ) project.is_active = False From 97e907d8d1e46b44886fc66a5df1e8f1b340b5db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Thu, 17 Mar 2022 14:05:10 +0100 Subject: [PATCH 126/293] changed aborted --- dds_web/api/project.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 7c8339f4c..8b7c08c81 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -96,7 +96,6 @@ def post(self): # Initial variable definition curr_date = dds_web.utils.current_time() - is_aborted = False delete_message = "" # Moving to Available @@ -290,13 +289,6 @@ def archive_project( """ # Check if valid status transition self.check_transition_possible(current_status=project.current_status, new_status="Archived") - - if project.current_status == "In Progress": - if project.has_been_available and not aborted: - raise DDSArgumentError( - "You cannot archive a project that has been made available previously. " - "Abort the project if you wish to proceed." - ) project.is_active = False # Deletes files (also commits session in the function - possibly refactor later) From 9f944db4a1e92ad14890164a7e4f8bd0cb1fa728 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Thu, 17 Mar 2022 14:15:50 +0100 Subject: [PATCH 127/293] Update dds_web/api/project.py --- dds_web/api/project.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 8b7c08c81..99aecce94 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -6,8 +6,6 @@ # Standard Library import http -from os import abort -from pyexpat import model from dds_web import database # Installed From 6255a2ca88b75a809e5374480d0aafb0d49bb39d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Thu, 17 Mar 2022 14:22:29 +0100 Subject: [PATCH 128/293] clarification in archived --- dds_web/api/project.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 8b7c08c81..11c949bdd 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -285,7 +285,8 @@ def archive_project( ): """Archive project: Make status Archived. - Only possible from Available and Expired. + Only possible from In Progress, Available and Expired. Optional aborted flag if something + has gone wrong. """ # Check if valid status transition self.check_transition_possible(current_status=project.current_status, new_status="Archived") From 78a0e4cc0613ea6358acfa8a5af09020eca27020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Thu, 17 Mar 2022 14:23:21 +0100 Subject: [PATCH 129/293] Update dds_web/api/project.py Co-authored-by: Anandashankar Anil --- dds_web/api/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 2f835e838..d0999abee 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -234,7 +234,7 @@ def expire_project( ) -> models.ProjectStatuses: """Expire project: Make status Expired. - Only possible Available. + Only possible from Available. """ # Check if valid status transition self.check_transition_possible(current_status=project.current_status, new_status="Expired") From 8055ed967a6b6e66c2354f4e2db15c4fa8a0b56d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Thu, 17 Mar 2022 14:45:02 +0100 Subject: [PATCH 130/293] fixed one test --- tests/test_project_status.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/tests/test_project_status.py b/tests/test_project_status.py index fbb7bd4fe..f58c75793 100644 --- a/tests/test_project_status.py +++ b/tests/test_project_status.py @@ -161,23 +161,9 @@ def test_aborted_project(module_client, boto3_session): project = project_row(project_id=project_id) assert file_in_db(test_dict=FIRST_NEW_FILE, project=project.id) - - new_status = {"new_status": "Archived"} - response = module_client.post( - tests.DDSEndpoint.PROJECT_STATUS, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(module_client), - query_string={"project": project_id}, - json=new_status, - ) - - assert response.status_code == http.HTTPStatus.BAD_REQUEST assert project.current_status == "In Progress" - assert ( - "You cannot archive a project that has been made available previously" - in response.json["message"] - ) - new_status["new_status"] = "Available" + new_status = {"new_status": "Available"} with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: response = module_client.post( tests.DDSEndpoint.PROJECT_STATUS, From b5d21e26146c3a43c6debd7a9c4e2d85d3cf2539 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Thu, 17 Mar 2022 15:15:13 +0100 Subject: [PATCH 131/293] add the in progress check again --- dds_web/api/project.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index d0999abee..6bf3c4e44 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -288,6 +288,11 @@ def archive_project( """ # Check if valid status transition self.check_transition_possible(current_status=project.current_status, new_status="Archived") + if project.current_status == "In Progress": + if not (project.has_been_available and aborted): + raise DDSArgumentError( + "Project cannot be archived from this status but can be aborted if it has ever been made available" + ) project.is_active = False # Deletes files (also commits session in the function - possibly refactor later) From 892b5d2e996a3fb35a3bb577e8f4ac6b342bb11f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Thu, 17 Mar 2022 15:21:27 +0100 Subject: [PATCH 132/293] readd the test I removed --- tests/test_project_status.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/test_project_status.py b/tests/test_project_status.py index f58c75793..a8110ff0f 100644 --- a/tests/test_project_status.py +++ b/tests/test_project_status.py @@ -161,9 +161,23 @@ def test_aborted_project(module_client, boto3_session): project = project_row(project_id=project_id) assert file_in_db(test_dict=FIRST_NEW_FILE, project=project.id) + + new_status = {"new_status": "Archived"} + response = module_client.post( + tests.DDSEndpoint.PROJECT_STATUS, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(module_client), + query_string={"project": project_id}, + json=new_status, + ) + + assert response.status_code == http.HTTPStatus.BAD_REQUEST assert project.current_status == "In Progress" + assert ( + "Project cannot be archived from this status but can be aborted if it has ever been made available" + in response.json["message"] + ) - new_status = {"new_status": "Available"} + new_status["new_status"] = "Available" with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: response = module_client.post( tests.DDSEndpoint.PROJECT_STATUS, From 8868468474c07534f720a3769e150f0813cb9d7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Thu, 17 Mar 2022 16:33:07 +0100 Subject: [PATCH 133/293] tests should be fixed now --- dds_web/api/project.py | 3 ++- tests/test_project_status.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 6bf3c4e44..39c0a64ae 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -291,7 +291,8 @@ def archive_project( if project.current_status == "In Progress": if not (project.has_been_available and aborted): raise DDSArgumentError( - "Project cannot be archived from this status but can be aborted if it has ever been made available" + "You cannot archive a project that has been made available previously. " + "Please abort the project if you wish to proceed." ) project.is_active = False diff --git a/tests/test_project_status.py b/tests/test_project_status.py index a8110ff0f..fbb7bd4fe 100644 --- a/tests/test_project_status.py +++ b/tests/test_project_status.py @@ -173,7 +173,7 @@ def test_aborted_project(module_client, boto3_session): assert response.status_code == http.HTTPStatus.BAD_REQUEST assert project.current_status == "In Progress" assert ( - "Project cannot be archived from this status but can be aborted if it has ever been made available" + "You cannot archive a project that has been made available previously" in response.json["message"] ) From b32002a16ae25e8af3b56adf944bf234e8c06be7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Thu, 17 Mar 2022 17:17:47 +0100 Subject: [PATCH 134/293] changed --- dds_web/api/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 39c0a64ae..953e832d1 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -289,7 +289,7 @@ def archive_project( # Check if valid status transition self.check_transition_possible(current_status=project.current_status, new_status="Archived") if project.current_status == "In Progress": - if not (project.has_been_available and aborted): + if project.has_been_available and not aborted: raise DDSArgumentError( "You cannot archive a project that has been made available previously. " "Please abort the project if you wish to proceed." From 1fecab90e123eed50ea371ad3ad52b38555664e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Thu, 17 Mar 2022 17:41:59 +0100 Subject: [PATCH 135/293] fixed return message for aborted --- dds_web/api/project.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 953e832d1..3e7cd19db 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -95,6 +95,7 @@ def post(self): # Initial variable definition curr_date = dds_web.utils.current_time() delete_message = "" + is_aborted = False # Moving to Available if new_status == "Available": @@ -136,7 +137,9 @@ def post(self): userobj=user.researchuser, mail_type="project_release", project=project ) - return_message = f"{project.public_id} updated to status {new_status}" + return_message = f"{project.public_id} updated to status {new_status}" + ( + " (aborted)" if new_status == "Archived" and is_aborted else "" + ) if new_status != "Available": return_message += delete_message + "." From d25ba470b0ce45cc6791e589166ceb6b673ed038 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Thu, 17 Mar 2022 17:42:45 +0100 Subject: [PATCH 136/293] Update dds_web/api/project.py Co-authored-by: Anandashankar Anil --- dds_web/api/project.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 3e7cd19db..250f57e6d 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -265,7 +265,8 @@ def delete_project(self, project: models.Project, current_time: datetime.datetim # Can only be Deleted if never made Available if project.has_been_available: raise DDSArgumentError( - "Project cannot be deleted if it has ever been made available, abort it instead." + "You cannot delete a project that has been made available previously. " + "Please abort the project if you wish to proceed." ) project.is_active = False From 5d390a086f8cea275d59d09dc483efdbfc6781ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Thu, 17 Mar 2022 17:46:25 +0100 Subject: [PATCH 137/293] black --- dds_web/api/project.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 250f57e6d..6c2b26ed5 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -265,8 +265,8 @@ def delete_project(self, project: models.Project, current_time: datetime.datetim # Can only be Deleted if never made Available if project.has_been_available: raise DDSArgumentError( - "You cannot delete a project that has been made available previously. " - "Please abort the project if you wish to proceed." + "You cannot delete a project that has been made available previously. " + "Please abort the project if you wish to proceed." ) project.is_active = False From 41b9a7d8011ad947b81ac76be31c9c07e512dbcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Thu, 17 Mar 2022 17:48:40 +0100 Subject: [PATCH 138/293] fix message in test! --- tests/test_project_status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_project_status.py b/tests/test_project_status.py index 6ccacba87..38f113a9b 100644 --- a/tests/test_project_status.py +++ b/tests/test_project_status.py @@ -454,7 +454,7 @@ def test_check_deadline_remains_same_when_made_available_again_after_going_to_in ) assert response.status_code == http.HTTPStatus.BAD_REQUEST assert ( - "Project cannot be deleted if it has ever been made available, abort it instead" + "You cannot delete a project that has been made available previously" in response.json["message"] ) assert project.current_status == "In Progress" From 3e6a9665d6b6f9018993c85b1ded45103c06d043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Thu, 17 Mar 2022 17:56:21 +0100 Subject: [PATCH 139/293] added uncaught exceptions that were in the previous version --- dds_web/api/project.py | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 6c2b26ed5..f8e6c81b5 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -270,12 +270,18 @@ def delete_project(self, project: models.Project, current_time: datetime.datetim ) project.is_active = False - # Deletes files (also commits session in the function - possibly refactor later) - RemoveContents().delete_project_contents(project=project) - self.rm_project_user_keys(project=project) + try: + # Deletes files (also commits session in the function - possibly refactor later) + RemoveContents().delete_project_contents(project=project) + self.rm_project_user_keys(project=project) + + # Delete metadata from project row + self.delete_project_info(proj=project) + except (TypeError, DatabaseError, DeletionError, BucketNotFoundError) as err: + flask.current_app.logger.exception(err) + db.session.rollback() + raise DeletionError(message="Server Error: Status was not updated") from err - # Delete metadata from project row - self.delete_project_info(proj=project) delete_message = ( f"\nAll files in project '{project.public_id}' deleted and project info cleared" ) @@ -300,15 +306,20 @@ def archive_project( ) project.is_active = False - # Deletes files (also commits session in the function - possibly refactor later) - RemoveContents().delete_project_contents(project=project) - delete_message = f"\nAll files in {project.public_id} deleted" - self.rm_project_user_keys(project=project) - - # Delete metadata from project row - if aborted: - project = self.delete_project_info(project) - delete_message += " and project info cleared" + try: + # Deletes files (also commits session in the function - possibly refactor later) + RemoveContents().delete_project_contents(project=project) + delete_message = f"\nAll files in {project.public_id} deleted" + self.rm_project_user_keys(project=project) + + # Delete metadata from project row + if aborted: + project = self.delete_project_info(project) + delete_message += " and project info cleared" + except (TypeError, DatabaseError, DeletionError, BucketNotFoundError) as err: + flask.current_app.logger.exception(err) + db.session.rollback() + raise DeletionError(message="Server Error: Status was not updated") from err return ( models.ProjectStatuses( From 318b7a25a36e64e98413ccb3d3424981e3977456 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Thu, 17 Mar 2022 17:58:20 +0100 Subject: [PATCH 140/293] removed unused import --- dds_web/api/project.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index f8e6c81b5..55ac9d79f 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -6,7 +6,6 @@ # Standard Library import http -from dds_web import database # Installed import flask_restful From 91df3468af6216907826a969758fd0ca860e8a2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Thu, 17 Mar 2022 21:50:07 +0100 Subject: [PATCH 141/293] tests in test_project_status fixed --- tests/test_project_status.py | 75 ++++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 29 deletions(-) diff --git a/tests/test_project_status.py b/tests/test_project_status.py index 38f113a9b..b9d814e4d 100644 --- a/tests/test_project_status.py +++ b/tests/test_project_status.py @@ -139,8 +139,8 @@ def test_set_project_to_deleted_from_in_progress(module_client, boto3_session): assert not project.project_user_keys -def test_aborted_project(module_client, boto3_session): - """Create a project and try to abort it""" +def test_archived_project(module_client, boto3_session): + """Create a project and archive it""" response = module_client.post( tests.DDSEndpoint.PROJECT_CREATE, @@ -170,26 +170,41 @@ def test_aborted_project(module_client, boto3_session): json=new_status, ) - assert response.status_code == http.HTTPStatus.BAD_REQUEST - assert project.current_status == "In Progress" - assert ( - "You cannot archive a project that has been made available previously" - in response.json["message"] - ) + assert response.status_code == http.HTTPStatus.OK + assert project.current_status == "Archived" - new_status["new_status"] = "Available" - with unittest.mock.patch.object(flask_mail.Mail, "send") as mock_mail_send: - response = module_client.post( - tests.DDSEndpoint.PROJECT_STATUS, - headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(module_client), - query_string={"project": project_id}, - json=new_status, - ) - # One mail sent for the partial token and one for project release - assert mock_mail_send.call_count == 2 + assert not max(project.project_statuses, key=lambda x: x.date_created).is_aborted + assert not file_in_db(test_dict=FIRST_NEW_FILE, project=project.id) + assert not project.project_user_keys + for field, value in vars(project).items(): + if field in fields_set_to_null: + assert value + assert project.researchusers + + +def test_aborted_project(module_client, boto3_session): + """Create a project and try to abort it""" + + response = module_client.post( + tests.DDSEndpoint.PROJECT_CREATE, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(module_client), + json=proj_data, + ) assert response.status_code == http.HTTPStatus.OK - assert project.current_status == "Available" + + project_id = response.json.get("project_id") + # add a file + response = module_client.post( + tests.DDSEndpoint.FILE_NEW, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(module_client), + query_string={"project": project_id}, + json=FIRST_NEW_FILE, + ) + + project = project_row(project_id=project_id) + + assert file_in_db(test_dict=FIRST_NEW_FILE, project=project.id) for field, value in vars(project).items(): if field in fields_set_to_null: @@ -198,8 +213,7 @@ def test_aborted_project(module_client, boto3_session): assert project.project_user_keys time.sleep(1) - new_status["new_status"] = "Archived" - new_status["is_aborted"] = True + new_status = {"new_status": "Archived", "is_aborted": True} response = module_client.post( tests.DDSEndpoint.PROJECT_STATUS, headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(module_client), @@ -303,10 +317,17 @@ def test_abort_from_in_progress_once_made_available(module_client, boto3_session assert not project.project_user_keys -def test_check_invalid_transitions_from_in_progress(module_client, test_project): +def test_check_invalid_transitions_from_in_progress(module_client, boto3_session): """Check all invalid transitions from In Progress""" - project_id = test_project + response = module_client.post( + tests.DDSEndpoint.PROJECT_CREATE, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(module_client), + json=proj_data, + ) + assert response.status_code == http.HTTPStatus.OK + + project_id = response.json.get("project_id") project = project_row(project_id=project_id) # In Progress to Expired @@ -334,12 +355,8 @@ def test_check_invalid_transitions_from_in_progress(module_client, test_project) json=new_status, ) - assert response.status_code == http.HTTPStatus.BAD_REQUEST - assert project.current_status == "In Progress" - assert ( - "You cannot archive a project that has been made available previously" - in response.json["message"] - ) + assert response.status_code == http.HTTPStatus.OK + assert project.current_status == "Archived" def test_set_project_to_available_valid_transition(module_client, test_project): From eadb20b8b05167d5a52f7112b528aa684b3bb73e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Thu, 17 Mar 2022 22:35:55 +0100 Subject: [PATCH 142/293] fixed tests --- tests/test_project_creation.py | 3 ++- tests/test_project_status.py | 40 +++++++++++++++++++--------------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/tests/test_project_creation.py b/tests/test_project_creation.py index 0b8b69ff4..b8b98aeef 100644 --- a/tests/test_project_creation.py +++ b/tests/test_project_creation.py @@ -8,6 +8,7 @@ from turtle import update import unittest import time +import os # Installed import pytest @@ -51,7 +52,7 @@ def create_unit_admins(num_admins, unit_id=1): new_admins.append( models.UnitUser( **{ - "username": "unit_admin_" + str(i), + "username": "unit_admin_" + os.urandom(4).hex(), "name": "Unit Admin " + str(i), "password": "password", "is_admin": True, diff --git a/tests/test_project_status.py b/tests/test_project_status.py index 07fbd215a..3bb64b30b 100644 --- a/tests/test_project_status.py +++ b/tests/test_project_status.py @@ -43,18 +43,13 @@ @pytest.fixture(scope="module") def test_project(module_client): """Create a shared test project""" - create_unit_admins(num_admins=2) - - current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() - assert current_unit_admins == 3 - with unittest.mock.patch.object(boto3.session.Session, "resource") as mock_session: response = module_client.post( tests.DDSEndpoint.PROJECT_CREATE, headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(module_client), json=proj_data, ) - project_id = response.json.get("project_id") + project_id = response.json.get("project_id") # add a file response = module_client.post( tests.DDSEndpoint.FILE_NEW, @@ -66,10 +61,21 @@ def test_project(module_client): return project_id -def test_submit_request_with_invalid_args(module_client, test_project): +def test_submit_request_with_invalid_args(module_client, boto3_session): """Submit status request with invalid arguments""" + create_unit_admins(num_admins=2) - project_id = test_project + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 + + response = module_client.post( + tests.DDSEndpoint.PROJECT_CREATE, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(module_client), + json=proj_data, + ) + assert response.status_code == http.HTTPStatus.OK + + project_id = response.json.get("project_id") project = project_row(project_id=project_id) response = module_client.post( @@ -94,10 +100,12 @@ def test_submit_request_with_invalid_args(module_client, test_project): def test_set_project_to_deleted_from_in_progress(module_client, boto3_session): """Create project and set status to deleted""" - create_unit_admins(num_admins=2) - current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() assert current_unit_admins == 3 + # create_unit_admins(num_admins=2) + + # current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + # assert current_unit_admins == 3 new_status = {"new_status": "Deleted"} response = module_client.post( @@ -151,9 +159,6 @@ def test_set_project_to_deleted_from_in_progress(module_client, boto3_session): def test_archived_project(module_client, boto3_session): """Create a project and archive it""" - - create_unit_admins(num_admins=2) - current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() assert current_unit_admins == 3 @@ -200,6 +205,8 @@ def test_archived_project(module_client, boto3_session): def test_aborted_project(module_client, boto3_session): """Create a project and try to abort it""" + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 response = module_client.post( tests.DDSEndpoint.PROJECT_CREATE, @@ -250,8 +257,6 @@ def test_aborted_project(module_client, boto3_session): def test_abort_from_in_progress_once_made_available(module_client, boto3_session): """Create project and abort it from In Progress after it has been made available""" - create_unit_admins(num_admins=2) - current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() assert current_unit_admins == 3 @@ -339,6 +344,8 @@ def test_abort_from_in_progress_once_made_available(module_client, boto3_session def test_check_invalid_transitions_from_in_progress(module_client, boto3_session): """Check all invalid transitions from In Progress""" + current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() + assert current_unit_admins == 3 response = module_client.post( tests.DDSEndpoint.PROJECT_CREATE, @@ -408,9 +415,6 @@ def test_set_project_to_available_valid_transition(module_client, test_project): def test_set_project_to_available_no_mail(module_client, boto3_session): """Set status to Available for test project, but skip sending mails""" - - create_unit_admins(num_admins=2) - current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() assert current_unit_admins == 3 From 989f22960698b61f1607bd5d3ce2679f622b1133 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Fri, 18 Mar 2022 08:44:08 +0100 Subject: [PATCH 143/293] Update dds_web/api/project.py --- dds_web/api/project.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 46b7518d2..4c879017f 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -5,7 +5,6 @@ #################################################################################################### # Standard Library -from email import message import http # Installed From 7772832de9b390055bd17b1895c1049735f686f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Fri, 18 Mar 2022 08:44:49 +0100 Subject: [PATCH 144/293] Update tests/test_project_creation.py --- tests/test_project_creation.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_project_creation.py b/tests/test_project_creation.py index b8b98aeef..8af8ff6ea 100644 --- a/tests/test_project_creation.py +++ b/tests/test_project_creation.py @@ -4,8 +4,6 @@ import http import datetime import json -from tokenize import Triple -from turtle import update import unittest import time import os From c70841ef3bbd92cea4e67d4872c36ffd87a5d8d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Fri, 18 Mar 2022 09:48:33 +0100 Subject: [PATCH 145/293] fixed product to project --- dds_web/api/user.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index 5b4aaadd0..af72b590c 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -384,14 +384,14 @@ def add_to_project(whom, project, role, send_email=True): AddUser.compose_and_send_email_to_user(whom, "project_release", project=project) flask.current_app.logger.debug( - f"{str(whom)} was given access to the {str(project)} as a {'Product Owner' if is_owner else 'Researcher'}." + f"{str(whom)} was given access to the {str(project)} as a {'Project Owner' if is_owner else 'Researcher'}." ) return { "status": http.HTTPStatus.OK, "message": ( f"{str(whom)} was given access to the " - f"{str(project)} as a {'Product Owner' if is_owner else 'Researcher'}. An e-mail notification has{' not ' if not send_email else ' '}been sent." + f"{str(project)} as a {'Project Owner' if is_owner else 'Researcher'}. An e-mail notification has{' not ' if not send_email else ' '}been sent." ), } From 51e086c9dbe56ca46561decef3f85006f3cee50e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Fri, 18 Mar 2022 10:38:52 +0100 Subject: [PATCH 146/293] Update tests/test_project_status.py Co-authored-by: Anandashankar Anil --- tests/test_project_status.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_project_status.py b/tests/test_project_status.py index 3bb64b30b..6f2382487 100644 --- a/tests/test_project_status.py +++ b/tests/test_project_status.py @@ -102,10 +102,6 @@ def test_set_project_to_deleted_from_in_progress(module_client, boto3_session): """Create project and set status to deleted""" current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() assert current_unit_admins == 3 - # create_unit_admins(num_admins=2) - - # current_unit_admins = models.UnitUser.query.filter_by(unit_id=1, is_admin=True).count() - # assert current_unit_admins == 3 new_status = {"new_status": "Deleted"} response = module_client.post( From 56bb708de295dfe6e6f12a7280f973812388ff16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Sun, 20 Mar 2022 13:45:33 +0100 Subject: [PATCH 147/293] return int, not str --- dds_web/api/files.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dds_web/api/files.py b/dds_web/api/files.py index 2ae2b9d34..56653c744 100644 --- a/dds_web/api/files.py +++ b/dds_web/api/files.py @@ -258,7 +258,7 @@ def get(self): "folder": False, } if show_size: - info.update({"size": dds_web.utils.format_byte_size(x[1])}) + info.update({"size": x[1]}) files_folders.append(info) if distinct_folders: for x in distinct_folders: @@ -269,7 +269,7 @@ def get(self): if show_size: folder_size = self.get_folder_size(project=project, folder_name=x) - info.update({"size": dds_web.utils.format_byte_size(folder_size)}) + info.update({"size": folder_size}) files_folders.append(info) return {"files_folders": files_folders} From 703effd53085de4f0a129eaa282f600751977230 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Sun, 20 Mar 2022 13:46:06 +0100 Subject: [PATCH 148/293] remove unused --- dds_web/utils.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/dds_web/utils.py b/dds_web/utils.py index 1022c2f57..ac964d829 100644 --- a/dds_web/utils.py +++ b/dds_web/utils.py @@ -329,19 +329,6 @@ def working_directory(path): os.chdir(current_path) -def format_byte_size(size): - """Take size in bytes and converts according to the size""" - suffixes = ["bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"] - - for suffix in suffixes: - if size >= 1000: - size /= 1000 - else: - break - - return f"{size:.2} {suffix}" if isinstance(size, float) else f"{size} {suffix}" - - def page_query(q): offset = 0 while True: From 62360ace9f7e695080b0dc4f6703cd43684b7ac9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Sun, 20 Mar 2022 14:56:40 +0100 Subject: [PATCH 149/293] always return size --- dds_web/api/project.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 4c879017f..edb9b72a9 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -424,13 +424,10 @@ def get(self): "Last updated": p.date_updated if p.date_updated else p.date_created, } - if ( - current_user.role == "Researcher" and p.current_status == "Available" - ) or current_user.role != "Researcher": - # Get proj size and update total size - proj_size = p.size - total_size += proj_size - project_info["Size"] = proj_size + # Get proj size and update total size + proj_size = p.size + total_size += proj_size + project_info["Size"] = proj_size if usage: proj_bhours, proj_cost = self.project_usage(project=p) @@ -454,11 +451,10 @@ def get(self): "usage": total_bhours_db, "cost": total_cost_db, }, + "total_size": total_size, + "always_show": current_user.role in ["Super Admin", "Unit Admin", "Unit Personnel"], } - if total_size or current_user.role in ["Unit Admin", "Unit Personnel"]: - return_info["total_size"] = total_size - return return_info @staticmethod From fc3f9075106b6d0f624de542a8f3b553bb5c4211 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Sun, 20 Mar 2022 15:11:35 +0100 Subject: [PATCH 150/293] only return usage if unit user --- dds_web/api/project.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index edb9b72a9..f5ab60c1f 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -446,14 +446,15 @@ def get(self): return_info = { "project_info": all_projects, - "total_usage": { - # return ByteHours - "usage": total_bhours_db, - "cost": total_cost_db, - }, "total_size": total_size, "always_show": current_user.role in ["Super Admin", "Unit Admin", "Unit Personnel"], } + if current_user.role in ["Super Admin", "Unit Admin", "Unit Personnel"]: + return_info["total_usage"] = { + # return ByteHours + "usage": total_bhours_db, + "cost": total_cost_db, + } return return_info From 59d759f180ab8257c566b2ca27aaba6b6514283e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Sun, 20 Mar 2022 15:50:38 +0100 Subject: [PATCH 151/293] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ad4e04e5..5cbe0974d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,3 +52,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - Make previous HOTP invalid at password reset ([#1054](https://github.com/ScilifelabDataCentre/dds_web/pull/1054)) - New endpoint for listing Units as Super Admin ([1060](https://github.com/ScilifelabDataCentre/dds_web/pull/1060)) - New endpoint for listing unit users as Super Admin ([#1059](https://github.com/ScilifelabDataCentre/dds_web/pull/1059)) +- Return int instead of string from files listing and only return usage info if right role () \ No newline at end of file From e9d3b1af6807b92837620d24e0e614e252a9a3d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Sun, 20 Mar 2022 15:52:16 +0100 Subject: [PATCH 152/293] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cbe0974d..d72faefa2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,4 +52,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - Make previous HOTP invalid at password reset ([#1054](https://github.com/ScilifelabDataCentre/dds_web/pull/1054)) - New endpoint for listing Units as Super Admin ([1060](https://github.com/ScilifelabDataCentre/dds_web/pull/1060)) - New endpoint for listing unit users as Super Admin ([#1059](https://github.com/ScilifelabDataCentre/dds_web/pull/1059)) -- Return int instead of string from files listing and only return usage info if right role () \ No newline at end of file +- Return int instead of string from files listing and only return usage info if right role ([#1070](https://github.com/ScilifelabDataCentre/dds_web/pull/1070)) \ No newline at end of file From d2fde798d7ae1657d65628faccfee720e7eedd50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Sun, 20 Mar 2022 15:52:55 +0100 Subject: [PATCH 153/293] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d72faefa2..4afa5f3b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,4 +52,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - Make previous HOTP invalid at password reset ([#1054](https://github.com/ScilifelabDataCentre/dds_web/pull/1054)) - New endpoint for listing Units as Super Admin ([1060](https://github.com/ScilifelabDataCentre/dds_web/pull/1060)) - New endpoint for listing unit users as Super Admin ([#1059](https://github.com/ScilifelabDataCentre/dds_web/pull/1059)) -- Return int instead of string from files listing and only return usage info if right role ([#1070](https://github.com/ScilifelabDataCentre/dds_web/pull/1070)) \ No newline at end of file +- Return int instead of string from files listing and only return usage info if right role ([#1070](https://github.com/ScilifelabDataCentre/dds_web/pull/1070)) From f917d1976e6311b78dfda71a86a2723913bf4f99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Sun, 20 Mar 2022 19:09:28 +0100 Subject: [PATCH 154/293] Update dds_web/web/user.py --- dds_web/web/user.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dds_web/web/user.py b/dds_web/web/user.py index db913ceaf..39fc0db87 100644 --- a/dds_web/web/user.py +++ b/dds_web/web/user.py @@ -343,6 +343,7 @@ def request_reset_password(): ).one_or_none() if ongoing_password_reset: ongoing_password_reset.issued = dds_web.utils.current_time() + ongoing_password_reset.valid = True else: new_password_reset = models.PasswordReset( user=email.user, email=email.email, issued=dds_web.utils.current_time() From 5f1381453356a0356f5151f770bd6ed135b2eef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Mon, 21 Mar 2022 08:32:39 +0100 Subject: [PATCH 155/293] authentication required message after password reset --- dds_web/database/models.py | 1 + dds_web/security/auth.py | 11 +++++++++++ dds_web/web/user.py | 1 + .../versions/1256117ad629_add_password_reset.py | 1 + 4 files changed, 14 insertions(+) diff --git a/dds_web/database/models.py b/dds_web/database/models.py index 8ce24741b..480225c66 100644 --- a/dds_web/database/models.py +++ b/dds_web/database/models.py @@ -779,6 +779,7 @@ class PasswordReset(db.Model): email = db.Column(db.String(254), unique=True, nullable=False) issued = db.Column(db.DateTime(), unique=False, nullable=False) + changed = db.Column(db.DateTime(), unique=False, nullable=True) valid = db.Column(db.Boolean, unique=False, nullable=False, default=True) diff --git a/dds_web/security/auth.py b/dds_web/security/auth.py index fe2aac33f..11c3a64b5 100644 --- a/dds_web/security/auth.py +++ b/dds_web/security/auth.py @@ -195,6 +195,17 @@ def verify_token(token): user = __user_from_subject(subject=claims.get("sub")) + if user.password_reset: + token_expired = claims.get("exp") + token_issued = datetime.datetime.fromtimestamp(token_expired) - MFA_EXPIRES_IN + password_reset_row = user.password_reset[0] + if not password_reset_row.valid and password_reset_row.changed > token_issued: + raise AuthenticationError( + message=( + "Password reset performed after last authentication. " + "Start a new authenticated session to proceed." + ) + ) return __handle_multi_factor_authentication( user=user, mfa_auth_time_string=claims.get("mfa_auth_time") ) diff --git a/dds_web/web/user.py b/dds_web/web/user.py index 39fc0db87..f8f68ee48 100644 --- a/dds_web/web/user.py +++ b/dds_web/web/user.py @@ -420,6 +420,7 @@ def reset_password(token): # Set password reset row as invalid password_reset_row.valid = False + password_reset_row.changed = dds_web.utils.current_time() db.session.commit() flask.flash("Your password has been updated! You are now able to log in.", "success") diff --git a/migrations/versions/1256117ad629_add_password_reset.py b/migrations/versions/1256117ad629_add_password_reset.py index 47474f9fd..fe3cba17c 100644 --- a/migrations/versions/1256117ad629_add_password_reset.py +++ b/migrations/versions/1256117ad629_add_password_reset.py @@ -24,6 +24,7 @@ def upgrade(): sa.Column("user_id", sa.String(length=50), nullable=True), sa.Column("email", sa.String(length=254), nullable=False), sa.Column("issued", sa.DateTime(), nullable=False), + sa.Column("changed", sa.DateTime(), nullable=True), sa.Column("valid", sa.Boolean(), nullable=False), sa.ForeignKeyConstraint(["user_id"], ["users.username"], ondelete="CASCADE"), sa.PrimaryKeyConstraint("id"), From 029171733af682d5d4c0f032e1ea58d583c6538f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 17 Mar 2022 14:34:47 +0100 Subject: [PATCH 156/293] Make sure remove_multiple never get more than 1000 entries --- dds_web/api/api_s3_connector.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dds_web/api/api_s3_connector.py b/dds_web/api/api_s3_connector.py index 81c4ff855..ac09bc3fd 100644 --- a/dds_web/api/api_s3_connector.py +++ b/dds_web/api/api_s3_connector.py @@ -87,10 +87,12 @@ def remove_bucket(self, *args, **kwargs): @bucket_must_exists def remove_multiple(self, items, *args, **kwargs): """Removes all with prefix.""" - _ = self.resource.meta.client.delete_objects( - Bucket=self.project.bucket, - Delete={"Objects": [{"Key": x} for x in items]}, - ) + # s3 can only delete 1000 objects per request + for i in range(0, len(items), 1000): + _ = self.resource.meta.client.delete_objects( + Bucket=self.project.bucket, + Delete={"Objects": [{"Key": x} for x in items[i:i+1000]]}, + ) @bucket_must_exists def remove_one(self, file, *args, **kwargs): From 7918239f41a1268f2a3c96fdeacbac39f4fa918a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 17 Mar 2022 14:35:43 +0100 Subject: [PATCH 157/293] Split folder deletion into batches of 1000 files --- dds_web/api/api_s3_connector.py | 2 +- dds_web/api/files.py | 88 ++++++++++++++++----------------- 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/dds_web/api/api_s3_connector.py b/dds_web/api/api_s3_connector.py index ac09bc3fd..7774eeae4 100644 --- a/dds_web/api/api_s3_connector.py +++ b/dds_web/api/api_s3_connector.py @@ -91,7 +91,7 @@ def remove_multiple(self, items, *args, **kwargs): for i in range(0, len(items), 1000): _ = self.resource.meta.client.delete_objects( Bucket=self.project.bucket, - Delete={"Objects": [{"Key": x} for x in items[i:i+1000]]}, + Delete={"Objects": [{"Key": x} for x in items[i : i + 1000]]}, ) @bucket_must_exists diff --git a/dds_web/api/files.py b/dds_web/api/files.py index 56653c744..47e5241e5 100644 --- a/dds_web/api/files.py +++ b/dds_web/api/files.py @@ -414,6 +414,7 @@ def delete_multiple(self, project, files): try: s3conn.remove_one(file=name_in_bucket) except (BucketNotFoundError, botocore.client.ClientError) as err: + print(err) db.session.rollback() not_removed_dict[x] = str(err) continue @@ -467,7 +468,6 @@ def delete(self): """Delete folder(s).""" # Verify project ID and access project = project_schemas.ProjectRequiredSchema().load(flask.request.args) - # Verify project status ok for deletion check_eligibility_for_deletion( status=project.current_status, has_been_available=project.has_been_available @@ -476,42 +476,47 @@ def delete(self): # Remove folder(s) not_removed_dict, not_exist_list = ({}, []) with ApiS3Connector(project=project) as s3conn: - for x in flask.request.json: + for folder_name in flask.request.json: # Get all files in the folder try: - in_db, objects_to_delete = self.delete_folder(project=project, folder=x) - if not in_db: - not_exist_list.append(x) + files = self.get_files_for_deletion(project=project, folder=folder_name) + if not files: + not_exist_list.append(folder_name) raise FileNotFoundError( "Could not find the specified folder in the database." ) except (sqlalchemy.exc.SQLAlchemyError, FileNotFoundError) as err: - db.session.rollback() - not_removed_dict[x] = str(err) + not_removed_dict[folder_name] = str(err) continue - # Delete from s3 - try: - s3conn.remove_multiple(items=objects_to_delete) - except botocore.client.ClientError as err: - db.session.rollback() - not_removed_dict[x] = str(err) - continue - - # Commit to db if no error so far - try: - db.session.commit() - except sqlalchemy.exc.SQLAlchemyError as err: - db.session.rollback() - not_removed_dict[x] = str(err) - continue + # S3 can only delete 1000 files per request + # The deletion will thus be divided into parts of at most 1000 files + for i in range(0, len(files), 1000): + # Delete from s3 + bucket_names = tuple(entry.name_in_bucket for entry in files[i : i + 1000]) + try: + s3conn.remove_multiple(items=bucket_names) + except botocore.client.ClientError as err: + not_removed_dict[folder_name] = str(err) + break + + self.delete_files(files[i : i + 1000]) + project.date_updated = dds_web.utils.current_time() + # Commit to db if no error so far + try: + db.session.commit() + except sqlalchemy.exc.SQLAlchemyError as err: + db.session.rollback() + flask.current_app.logger.error( + "Files deleted in S3 but not in db. The entries must be synchronised!" + ) + not_removed_dict[folder_name] = str(err) + break return {"not_removed": not_removed_dict, "not_exists": not_exist_list} - def delete_folder(self, project, folder): - """Delete all items in folder""" - exists = False - names_in_bucket = [] + def get_files_for_deletion(self, project: str, folder: str): + """Get all file entries from db""" if folder[-1] == "/": folder = folder[:-1] re_folder = re.escape(folder) @@ -529,28 +534,23 @@ def delete_folder(self, project, folder): ) .all() ) - except sqlalchemy.exc.SQLAlchemyError as err: raise DatabaseError(message=str(err)) from err - if files: - exists = True - for x in files: - # get current version - current_file_version = models.Version.query.filter( - sqlalchemy.and_( - models.Version.active_file == sqlalchemy.func.binary(x.id), - models.Version.time_deleted.is_(None), - ) - ).first() - current_file_version.time_deleted = dds_web.utils.current_time() - - # Delete file and update project size - names_in_bucket.append(x.name_in_bucket) - db.session.delete(x) - project.date_updated = dds_web.utils.current_time() + return files - return exists, names_in_bucket + def delete_files(self, files: list): + """Prepare queries in the db session for deletion of files in the database.""" + for entry in files: + # get current version + current_file_version = models.Version.query.filter( + sqlalchemy.and_( + models.Version.active_file == sqlalchemy.func.binary(entry.id), + models.Version.time_deleted.is_(None), + ) + ).first() + current_file_version.time_deleted = dds_web.utils.current_time() + db.session.delete(entry) class FileInfo(flask_restful.Resource): From e3268f14ec8202523de611e5ddb2934a3cc2fdf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Fri, 18 Mar 2022 10:55:28 +0100 Subject: [PATCH 158/293] Rename variable from x to entry --- dds_web/api/files.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/dds_web/api/files.py b/dds_web/api/files.py index 47e5241e5..ffde7ffe1 100644 --- a/dds_web/api/files.py +++ b/dds_web/api/files.py @@ -391,32 +391,30 @@ def delete_multiple(self, project, files): not_removed_dict, not_exist_list = ({}, []) with ApiS3Connector(project=project) as s3conn: - # Delete each file - for x in files: + for entry in files: # Delete from db try: - name_in_bucket = self.delete_one(project=project, filename=x) + name_in_bucket = self.delete_one(project=project, filename=entry) if not name_in_bucket: raise DatabaseError( message="Remote file name not found.", pass_message=True ) except FileNotFoundError: db.session.rollback() - not_exist_list.append(x) + not_exist_list.append(entry) continue except (sqlalchemy.exc.SQLAlchemyError, DatabaseError) as err: db.session.rollback() - not_removed_dict[x] = str(err) + not_removed_dict[entry] = str(err) continue # Remove from s3 bucket try: s3conn.remove_one(file=name_in_bucket) except (BucketNotFoundError, botocore.client.ClientError) as err: - print(err) db.session.rollback() - not_removed_dict[x] = str(err) + not_removed_dict[entry] = str(err) continue # Commit to db if ok @@ -424,7 +422,7 @@ def delete_multiple(self, project, files): db.session.commit() except sqlalchemy.exc.SQLAlchemyError as err: db.session.rollback() - not_removed_dict[x] = str(err) + not_removed_dict[entry] = str(err) continue return not_removed_dict, not_exist_list From a29bb0e1d1160da2040aea0b2a61b77f703bcc31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Fri, 18 Mar 2022 10:56:31 +0100 Subject: [PATCH 159/293] Include type of fail (s3/db), do not include type in variable name --- dds_web/api/files.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/dds_web/api/files.py b/dds_web/api/files.py index ffde7ffe1..6fc12dcc1 100644 --- a/dds_web/api/files.py +++ b/dds_web/api/files.py @@ -472,20 +472,17 @@ def delete(self): ) # Remove folder(s) - not_removed_dict, not_exist_list = ({}, []) + not_removed, not_exist = ({}, []) + fail_type = None with ApiS3Connector(project=project) as s3conn: for folder_name in flask.request.json: # Get all files in the folder - try: - files = self.get_files_for_deletion(project=project, folder=folder_name) - if not files: - not_exist_list.append(folder_name) - raise FileNotFoundError( - "Could not find the specified folder in the database." - ) - except (sqlalchemy.exc.SQLAlchemyError, FileNotFoundError) as err: - not_removed_dict[folder_name] = str(err) - continue + files = self.get_files_for_deletion(project=project, folder=folder_name) + if not files: + not_exist.append(folder_name) + raise FileNotFoundError( + "Could not find the specified folder in the database." + ) # S3 can only delete 1000 files per request # The deletion will thus be divided into parts of at most 1000 files @@ -495,7 +492,8 @@ def delete(self): try: s3conn.remove_multiple(items=bucket_names) except botocore.client.ClientError as err: - not_removed_dict[folder_name] = str(err) + not_removed[folder_name] = str(err) + fail_type = "s3" break self.delete_files(files[i : i + 1000]) @@ -508,10 +506,14 @@ def delete(self): flask.current_app.logger.error( "Files deleted in S3 but not in db. The entries must be synchronised!" ) - not_removed_dict[folder_name] = str(err) + not_removed[folder_name] = str(err) + fail_type = "db" break - return {"not_removed": not_removed_dict, "not_exists": not_exist_list} + return {"not_removed": not_removed, + "fail_type": fail_type, + "not_exists": not_exist, + "nr_deleted": len(files) if not not_removed else i} def get_files_for_deletion(self, project: str, folder: str): """Get all file entries from db""" From ca938fc64fb5150ee0c75fb92a87392c22c4bc5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Fri, 18 Mar 2022 11:45:27 +0100 Subject: [PATCH 160/293] Correct handling of failed folders --- dds_web/api/files.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dds_web/api/files.py b/dds_web/api/files.py index 6fc12dcc1..c89bfe19a 100644 --- a/dds_web/api/files.py +++ b/dds_web/api/files.py @@ -1,6 +1,6 @@ """Files module.""" -#################################################################################################### +################################################################################################### # IMPORTS ################################################################################ IMPORTS # #################################################################################################### @@ -480,9 +480,7 @@ def delete(self): files = self.get_files_for_deletion(project=project, folder=folder_name) if not files: not_exist.append(folder_name) - raise FileNotFoundError( - "Could not find the specified folder in the database." - ) + continue # S3 can only delete 1000 files per request # The deletion will thus be divided into parts of at most 1000 files From 95566828d935c0fce0cdbae22b6cba88aba60cdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Fri, 18 Mar 2022 13:03:35 +0100 Subject: [PATCH 161/293] Fix logic for folder deletion, catch pymysql.err.OperationalError --- dds_web/api/files.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/dds_web/api/files.py b/dds_web/api/files.py index c89bfe19a..8292bd0d0 100644 --- a/dds_web/api/files.py +++ b/dds_web/api/files.py @@ -9,11 +9,12 @@ import re # Installed -import flask_restful +import botocore import flask +import flask_restful +import pymysql import sqlalchemy import werkzeug -import botocore # Own modules import dds_web.utils @@ -483,7 +484,7 @@ def delete(self): continue # S3 can only delete 1000 files per request - # The deletion will thus be divided into parts of at most 1000 files + # The deletion will thus be divided into batches of at most 1000 files for i in range(0, len(files), 1000): # Delete from s3 bucket_names = tuple(entry.name_in_bucket for entry in files[i : i + 1000]) @@ -498,13 +499,19 @@ def delete(self): project.date_updated = dds_web.utils.current_time() # Commit to db if no error so far try: + self.queue_file_entry_deletion(files[i : i + 1000]) + project.date_updated = dds_web.utils.current_time() db.session.commit() - except sqlalchemy.exc.SQLAlchemyError as err: + except (sqlalchemy.exc.SQLAlchemyError, pymysql.err.OperationalError) as err: db.session.rollback() flask.current_app.logger.error( "Files deleted in S3 but not in db. The entries must be synchronised!" ) - not_removed[folder_name] = str(err) + if type(err) == pymysql.err.OperationalError: + err_msg = "Database malfunction." + else: + err_msg = str(err) + not_removed[folder_name] = err_msg fail_type = "db" break @@ -537,7 +544,7 @@ def get_files_for_deletion(self, project: str, folder: str): return files - def delete_files(self, files: list): + def queue_file_entry_deletion(self, files: list): """Prepare queries in the db session for deletion of files in the database.""" for entry in files: # get current version From f61d452e386adadd1c4cf034b29c4f7959d39576 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Fri, 18 Mar 2022 13:40:17 +0100 Subject: [PATCH 162/293] Use batch_size, correct exception --- dds_web/api/files.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/dds_web/api/files.py b/dds_web/api/files.py index 8292bd0d0..821f6b997 100644 --- a/dds_web/api/files.py +++ b/dds_web/api/files.py @@ -12,7 +12,6 @@ import botocore import flask import flask_restful -import pymysql import sqlalchemy import werkzeug @@ -485,9 +484,10 @@ def delete(self): # S3 can only delete 1000 files per request # The deletion will thus be divided into batches of at most 1000 files - for i in range(0, len(files), 1000): + batch_size: int = 1000 + for i in range(0, len(files), batch_size): # Delete from s3 - bucket_names = tuple(entry.name_in_bucket for entry in files[i : i + 1000]) + bucket_names = tuple(entry.name_in_bucket for entry in files[i : i + batch_size]) try: s3conn.remove_multiple(items=bucket_names) except botocore.client.ClientError as err: @@ -495,19 +495,17 @@ def delete(self): fail_type = "s3" break - self.delete_files(files[i : i + 1000]) - project.date_updated = dds_web.utils.current_time() # Commit to db if no error so far try: - self.queue_file_entry_deletion(files[i : i + 1000]) + self.queue_file_entry_deletion(files[i : i + batch_size]) project.date_updated = dds_web.utils.current_time() db.session.commit() - except (sqlalchemy.exc.SQLAlchemyError, pymysql.err.OperationalError) as err: + except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err: db.session.rollback() flask.current_app.logger.error( "Files deleted in S3 but not in db. The entries must be synchronised!" ) - if type(err) == pymysql.err.OperationalError: + if isinstance(err, sqlalchemy.exc.OperationalError): err_msg = "Database malfunction." else: err_msg = str(err) From 2e3ab3c63627718ce04827a9b35676c1c72b38f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Fri, 18 Mar 2022 13:44:03 +0100 Subject: [PATCH 163/293] Changelog note --- CHANGELOG.md | 1 + dds_web/api/files.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4afa5f3b5..2b56b5d5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,3 +53,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - New endpoint for listing Units as Super Admin ([1060](https://github.com/ScilifelabDataCentre/dds_web/pull/1060)) - New endpoint for listing unit users as Super Admin ([#1059](https://github.com/ScilifelabDataCentre/dds_web/pull/1059)) - Return int instead of string from files listing and only return usage info if right role ([#1070](https://github.com/ScilifelabDataCentre/dds_web/pull/1070)) +- Batch deletion of files (breaking atomicity) ([#1067](https://github.com/ScilifelabDataCentre/dds_web/pull/1067)) diff --git a/dds_web/api/files.py b/dds_web/api/files.py index 821f6b997..87131a0f5 100644 --- a/dds_web/api/files.py +++ b/dds_web/api/files.py @@ -1,6 +1,6 @@ """Files module.""" -################################################################################################### +#################################################################################################### # IMPORTS ################################################################################ IMPORTS # #################################################################################################### From 4aa5c4ed8b1520aaaaa0efd52633b66143c6ac71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Fri, 18 Mar 2022 13:46:10 +0100 Subject: [PATCH 164/293] Black formatting --- dds_web/api/files.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/dds_web/api/files.py b/dds_web/api/files.py index 87131a0f5..d05d9d0bd 100644 --- a/dds_web/api/files.py +++ b/dds_web/api/files.py @@ -487,7 +487,9 @@ def delete(self): batch_size: int = 1000 for i in range(0, len(files), batch_size): # Delete from s3 - bucket_names = tuple(entry.name_in_bucket for entry in files[i : i + batch_size]) + bucket_names = tuple( + entry.name_in_bucket for entry in files[i : i + batch_size] + ) try: s3conn.remove_multiple(items=bucket_names) except botocore.client.ClientError as err: @@ -513,10 +515,12 @@ def delete(self): fail_type = "db" break - return {"not_removed": not_removed, - "fail_type": fail_type, - "not_exists": not_exist, - "nr_deleted": len(files) if not not_removed else i} + return { + "not_removed": not_removed, + "fail_type": fail_type, + "not_exists": not_exist, + "nr_deleted": len(files) if not not_removed else i, + } def get_files_for_deletion(self, project: str, folder: str): """Get all file entries from db""" From 8b4963d6e76663d5a6ea1e8b6c0bfeb27eadd17d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Mon, 21 Mar 2022 09:57:57 +0100 Subject: [PATCH 165/293] Make batch size configurable --- dds_web/api/api_s3_connector.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dds_web/api/api_s3_connector.py b/dds_web/api/api_s3_connector.py index 7774eeae4..ad1a6bccb 100644 --- a/dds_web/api/api_s3_connector.py +++ b/dds_web/api/api_s3_connector.py @@ -85,13 +85,13 @@ def remove_bucket(self, *args, **kwargs): bucket = None @bucket_must_exists - def remove_multiple(self, items, *args, **kwargs): + def remove_multiple(self, items, batch_size: int = 1000, *args, **kwargs): """Removes all with prefix.""" # s3 can only delete 1000 objects per request - for i in range(0, len(items), 1000): + for i in range(0, len(items), batch_size): _ = self.resource.meta.client.delete_objects( Bucket=self.project.bucket, - Delete={"Objects": [{"Key": x} for x in items[i : i + 1000]]}, + Delete={"Objects": [{"Key": x} for x in items[i : i + batch_size]]}, ) @bucket_must_exists From aaa1c3ebfcabc418bbb09717e417c61dec78e864 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Mon, 21 Mar 2022 09:59:27 +0100 Subject: [PATCH 166/293] Add batch_size argument --- dds_web/api/files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/api/files.py b/dds_web/api/files.py index d05d9d0bd..20a307c2e 100644 --- a/dds_web/api/files.py +++ b/dds_web/api/files.py @@ -491,7 +491,7 @@ def delete(self): entry.name_in_bucket for entry in files[i : i + batch_size] ) try: - s3conn.remove_multiple(items=bucket_names) + s3conn.remove_multiple(items=bucket_names, batch_size=batch_size) except botocore.client.ClientError as err: not_removed[folder_name] = str(err) fail_type = "s3" From fb66bab4b7c877869feadd7d8dcd7539a514cc3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Mon, 21 Mar 2022 11:08:38 +0100 Subject: [PATCH 167/293] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4afa5f3b5..8def5cd84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,3 +53,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - New endpoint for listing Units as Super Admin ([1060](https://github.com/ScilifelabDataCentre/dds_web/pull/1060)) - New endpoint for listing unit users as Super Admin ([#1059](https://github.com/ScilifelabDataCentre/dds_web/pull/1059)) - Return int instead of string from files listing and only return usage info if right role ([#1070](https://github.com/ScilifelabDataCentre/dds_web/pull/1070)) +- Change token expiration time to 7 days (168 hours) () \ No newline at end of file From 8c98b0607d24901363d9a952f308b8509caeae97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Mon, 21 Mar 2022 11:09:23 +0100 Subject: [PATCH 168/293] changelog lint --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8def5cd84..3746de9d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,4 +53,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - New endpoint for listing Units as Super Admin ([1060](https://github.com/ScilifelabDataCentre/dds_web/pull/1060)) - New endpoint for listing unit users as Super Admin ([#1059](https://github.com/ScilifelabDataCentre/dds_web/pull/1059)) - Return int instead of string from files listing and only return usage info if right role ([#1070](https://github.com/ScilifelabDataCentre/dds_web/pull/1070)) -- Change token expiration time to 7 days (168 hours) () \ No newline at end of file +- Change token expiration time to 7 days (168 hours) () From c012e7594c96f197744bc6c2ce417907b7cf6dbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Mon, 21 Mar 2022 11:10:16 +0100 Subject: [PATCH 169/293] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3746de9d4..dc8c99158 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,4 +53,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - New endpoint for listing Units as Super Admin ([1060](https://github.com/ScilifelabDataCentre/dds_web/pull/1060)) - New endpoint for listing unit users as Super Admin ([#1059](https://github.com/ScilifelabDataCentre/dds_web/pull/1059)) - Return int instead of string from files listing and only return usage info if right role ([#1070](https://github.com/ScilifelabDataCentre/dds_web/pull/1070)) -- Change token expiration time to 7 days (168 hours) () +- Change token expiration time to 7 days (168 hours) ([#1061](https://github.com/ScilifelabDataCentre/dds_web/pull/1061)) From b5cb26cf8528996549176ff775301357aff52cdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Mon, 21 Mar 2022 12:30:32 +0100 Subject: [PATCH 170/293] boolean to tinyint --- migrations/versions/1256117ad629_add_password_reset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/versions/1256117ad629_add_password_reset.py b/migrations/versions/1256117ad629_add_password_reset.py index fe3cba17c..8cd23736d 100644 --- a/migrations/versions/1256117ad629_add_password_reset.py +++ b/migrations/versions/1256117ad629_add_password_reset.py @@ -25,7 +25,7 @@ def upgrade(): sa.Column("email", sa.String(length=254), nullable=False), sa.Column("issued", sa.DateTime(), nullable=False), sa.Column("changed", sa.DateTime(), nullable=True), - sa.Column("valid", sa.Boolean(), nullable=False), + sa.Column("valid", mysql.TINYINT(display_width=1), nullable=False), sa.ForeignKeyConstraint(["user_id"], ["users.username"], ondelete="CASCADE"), sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("email"), From 7fc639c9a19fcabe7360a826a05f045e2b23b435 Mon Sep 17 00:00:00 2001 From: Zishan Mirza Date: Mon, 21 Mar 2022 12:22:28 +0000 Subject: [PATCH 171/293] Update text This patch updates the text. Signed-off-by: Zishan Mirza --- dds_web/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/utils.py b/dds_web/utils.py index ac964d829..a43bacbb1 100644 --- a/dds_web/utils.py +++ b/dds_web/utils.py @@ -81,7 +81,7 @@ def email_taken(indata): """Validator - verify that email is taken.""" if not email_in_db(email=indata): raise marshmallow.validate.ValidationError( - "There is no account with that email. To get an account, you need an invitation." + "If the email is connected to a user within the DDS, you should receive an email with the password reset instructions." ) From d9719fa1eafa3e6c1a4cdaa844266c973604c79e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Mon, 21 Mar 2022 14:26:40 +0100 Subject: [PATCH 172/293] update adr --- ADR.md | 97 ++++++++++++++++++++++++++++++++- dds_web/api/api_s3_connector.py | 2 +- 2 files changed, 96 insertions(+), 3 deletions(-) diff --git a/ADR.md b/ADR.md index f71d44f03..cac2cdc31 100644 --- a/ADR.md +++ b/ADR.md @@ -1,7 +1,9 @@ # Architecture Decision Record (ADR) -# 1. Framework: Flask +---- +# 1. Framework: Flask +__Date/year:__ 2019/2020 ## Alternatives and comparisons ### Tornado @@ -41,7 +43,10 @@ Since Flask is flexible and simple, extensions provide a large variety of functionalities including REST API support, it has an integrated testing system and there is more online support than for Tornado, **Flask** was chosen as the better option for the Data Delivery System framework. The built-in asynchronicity in Tornado is not an important feature since the system will not be used by thousands of users at a given time. +----- + # 2. Database: MariaDB +__Date/year:__ 2019/2020 The initial database used under development was the non-relational CouchDB. As development progressed, it became of interest to investigate whether this was the best approach or whether a relational database was better for the systems purposes. @@ -72,7 +77,10 @@ The main motivation behind choosing a **relational database** is that we are for **MariaDB** was chosen (rather than e.g. MySQL) as the relational database because of it’s query performance and that it provides many useful features not available in other relational databases. +----- + # 3. Compression algorithm: ZStandard +__Date/year:__ 2019/2020 GNU zip (Gzip) is the most popular compression algorithm and is suitable for compression of data streams, is supported by all browsers and comes as a standard in all major web servers. However, while gzip provides a good compression ratio (original/compressed size), it is very slow compared to other algorithms. @@ -82,7 +90,10 @@ The encryption speed using ChaCha20-Poly1305 (in this case tested on a 109 MB fi Since Zstandard gave approximately the same compression ratio in a fraction of the time, **Zstandard** was chosen as the algorithm to be implemented within the Data Delivery System. +---- + # 4. In-transit vs local encryption: Local (for now, looking at options) +__Date/year:__ 2019/2020 The most efficient way of delivering the data to the owners would be to perform encryption- and decryption in-transit, thereby decreasing the amount of memory required locally and possibly the total delivery time. However, there have been some difficulties finding a working solution for this, including that the `crypt4gh` Python package used in the beginning of the development did not support it. @@ -93,7 +104,10 @@ On further investigation and contact with Safespring, we learned: - All users of the Safespring backup service perform encryption on their own and handle the keys themselves. - Due to this, the encryption will be performed locally before upload to the S3 storage. +----- + # 5. Encryption Algorithm: ChaCha20-Poly1305 +__Date/year:__ 2019/2020 The new encryption format standard for genomics and health related data is Crypt4GH, developed by the Global Alliance for Genomics and Health and first released in 2019. The general encryption standard, however, is AES of which AES-GCM is an authenticated encryption mode. AES is currently (since 2001) the block cipher Rijndael. ChaCha20 is a stream cipher and is used within the Crypt4GH format. The most secure, efficient and generally appropriate format and algorithm should be implemented within the Data Delivery System. @@ -203,7 +217,10 @@ Files larger than 256 GiB will need to be partitioned, however the number of par Due to this, **ChaCha20-Poly1305** was chosen as the encryption algorithm within the Data Delivery System. +---- + # 6. File chunk size: 64 KiB +__Date/year:__ 2019/2020 Compression and encryption is performed in a streamed manner to avoid reading large files completely in memory. This is done by reading the file in chunks, and the size of the chunks affect the speed of the algorithms, their memory usage, and the final size of the compressed and encrypted files. @@ -215,7 +232,10 @@ To find the optimal chunk size, a 33 GB file was compressed and encrypted using The Data Delivery System will read the files in 64 KiB chunks. +----- + # 7. File integrity guarantee: Nonce incrementation +__Date/year:__ 2019/2020 As described in section 5. above, the Crypt4GH format encrypts the files in blocks of 64 KiB, after which each data blocks unique nonce, ciphertext and MAC are saved to the c4gh file. This guarantees the integrity of the data blocks, however it does not guarantee the integrity of the entire file, and it is therefore possible that some blocks are rearranged, duplicated or missing, without the recipient knowing. Although we have chosen to not use the Crypt4GH format within the delivery system, we do use the same encryption algorithm – ChaCha20-Poly1305 – and (since we cannot read huge files in memory) we have chosen to read the files in equally sized chunks. Therefore the integrity issue can potentially give huge problems for the delivery system. @@ -243,7 +263,10 @@ Due to this, no checksum verification is used during the upload. However, the fi Nonce incrementation will be used and no checksum verification will be performed during upload. +----- + # 8. Password Authentication: Argon2id +__Date/year:__ 2021 Argon2 is also available in two other versions. These are argon2d (strong GPU resistance) and argon2i (resistant to side-channel attacks). Argon2id is a combination of the two and is the recommended mode. @@ -253,8 +276,78 @@ The Data Delivery System will use [Argon2id](https://github.com/hynek/argon2-cff > The chosen parameters will be added here soon. -# 9. Requirements: No pinned versions +---- + +# 9. User roles: Super Admin, Unit Admin, Unit Personnel, Researcher +__Date:__ September 14th 2021 + +## Decision + +### Super Admin (DC) +* Manage: Add, Remove, Edit + * Unit (instances) + * Users + +### Unit Admin +* Unit Personnel Permissions +* Manage: Add, Add to project, Remove from project, Remove account, Change permissions + * Unit Admin + * Unit User + +### Unit Personnel +* Project Owner Permissons +* Upload +* Delete + +### Project Owner +* Research User Permissions +* Manage: Invite, Add to project, Remove from project, Remove account, Change permissions + * Project Owners + * Research Users + +### Research Account +* Remove own account +* List +* Download + +---- +# 10. HOTP as default +__Date:__ December 1st 2022 + +Initially, TOTP was implemented as the Two Factor Authentication. Authentication apps such as Authy or Google Authenticator could be set up and used to identify a user. However, due to some technical difficulties for some users, it was decided that we need to allow 2FA via mail as a default. + +## Decision +Use email 2FA (using HOTP) as a default. 2FA with authenticator apps (with TOTP) will be implemented _at some point_ and the users will be able to choose which method they want to use. + +---- +# 11. Structured logging for action log +__Date:__ January 12th 2022 + +Example: https://newrelic.com/blog/how-to-relic/python-structured-logging + +## Decision +Structured logging should be implemented on the action logging first - the logging which saves when a user tries to perform a specific action e.g. upload/download/list/auth/rm etc. + +The information required to be logged: username, action, result (failed/successful), time, project in which the action was attempted. + +When the action logs have been fixed we will discuss whether or not this will be implemented in the general logging as well, such as debugging and general system info. + + +---- + +# 12. Requirements: No pinned versions +__Date:__ March 1st 2022 ## Decisions We will not pin the requirement versions. If at some point something stops working we will look into it then and update the requirements then. This will simplify the installation for the users which is one of our priorities. + +---- + +# 13. No `--username` option +__Date:__ March 2nd 2022 + +Previously there was a `--username` option for all commands where the user could specify the username. + +## Decision +We will not have the `--username` option. When using `dds auth login` command, either the existing encrypted token will be used or the user will be prompted to fill in the username and password. \ No newline at end of file diff --git a/dds_web/api/api_s3_connector.py b/dds_web/api/api_s3_connector.py index ad1a6bccb..08400bcde 100644 --- a/dds_web/api/api_s3_connector.py +++ b/dds_web/api/api_s3_connector.py @@ -107,6 +107,6 @@ def generate_get_url(self, key): url = self.resource.meta.client.generate_presigned_url( "get_object", Params={"Bucket": self.project.bucket, "Key": key}, - ExpiresIn=3 * 24 * 60 * 60, + ExpiresIn=604800, # 7 days in seconds ) return url From f27c4f5858c0f8a98e98947329dbb69e6c52212d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Mon, 21 Mar 2022 14:27:36 +0100 Subject: [PATCH 173/293] lint? --- ADR.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ADR.md b/ADR.md index cac2cdc31..32c7736d3 100644 --- a/ADR.md +++ b/ADR.md @@ -350,4 +350,4 @@ __Date:__ March 2nd 2022 Previously there was a `--username` option for all commands where the user could specify the username. ## Decision -We will not have the `--username` option. When using `dds auth login` command, either the existing encrypted token will be used or the user will be prompted to fill in the username and password. \ No newline at end of file +We will not have the `--username` option. When using `dds auth login` command, either the existing encrypted token will be used or the user will be prompted to fill in the username and password. From 819f56033c11d8410a86c96af6ecb44411933a0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Mon, 21 Mar 2022 14:29:51 +0100 Subject: [PATCH 174/293] - instead of * --- ADR.md | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/ADR.md b/ADR.md index 32c7736d3..ac1b04d1b 100644 --- a/ADR.md +++ b/ADR.md @@ -284,31 +284,31 @@ __Date:__ September 14th 2021 ## Decision ### Super Admin (DC) -* Manage: Add, Remove, Edit - * Unit (instances) - * Users +- Manage: Add, Remove, Edit + - Unit (instances) + - Users ### Unit Admin -* Unit Personnel Permissions -* Manage: Add, Add to project, Remove from project, Remove account, Change permissions - * Unit Admin - * Unit User +- Unit Personnel Permissions +- Manage: Add, Add to project, Remove from project, Remove account, Change permissions + - Unit Admin + - Unit User ### Unit Personnel -* Project Owner Permissons -* Upload -* Delete +- Project Owner Permissons +- Upload +- Delete ### Project Owner -* Research User Permissions -* Manage: Invite, Add to project, Remove from project, Remove account, Change permissions - * Project Owners - * Research Users +- Research User Permissions +- Manage: Invite, Add to project, Remove from project, Remove account, Change permissions + - Project Owners + - Research Users ### Research Account -* Remove own account -* List -* Download +- Remove own account +- List +- Download ---- # 10. HOTP as default From 4ebd5b12fa8415f5b502bd32f70b5c075c36bfee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Mon, 21 Mar 2022 14:36:13 +0100 Subject: [PATCH 175/293] prettier --- ADR.md | 99 ++++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 61 insertions(+), 38 deletions(-) diff --git a/ADR.md b/ADR.md index ac1b04d1b..25463f3d5 100644 --- a/ADR.md +++ b/ADR.md @@ -1,9 +1,11 @@ # Architecture Decision Record (ADR) ----- +--- # 1. Framework: Flask -__Date/year:__ 2019/2020 + +**Date/year:** 2019/2020 + ## Alternatives and comparisons ### Tornado @@ -43,10 +45,11 @@ __Date/year:__ 2019/2020 Since Flask is flexible and simple, extensions provide a large variety of functionalities including REST API support, it has an integrated testing system and there is more online support than for Tornado, **Flask** was chosen as the better option for the Data Delivery System framework. The built-in asynchronicity in Tornado is not an important feature since the system will not be used by thousands of users at a given time. ------ +--- # 2. Database: MariaDB -__Date/year:__ 2019/2020 + +**Date/year:** 2019/2020 The initial database used under development was the non-relational CouchDB. As development progressed, it became of interest to investigate whether this was the best approach or whether a relational database was better for the systems purposes. @@ -77,10 +80,11 @@ The main motivation behind choosing a **relational database** is that we are for **MariaDB** was chosen (rather than e.g. MySQL) as the relational database because of it’s query performance and that it provides many useful features not available in other relational databases. ------ +--- # 3. Compression algorithm: ZStandard -__Date/year:__ 2019/2020 + +**Date/year:** 2019/2020 GNU zip (Gzip) is the most popular compression algorithm and is suitable for compression of data streams, is supported by all browsers and comes as a standard in all major web servers. However, while gzip provides a good compression ratio (original/compressed size), it is very slow compared to other algorithms. @@ -90,10 +94,11 @@ The encryption speed using ChaCha20-Poly1305 (in this case tested on a 109 MB fi Since Zstandard gave approximately the same compression ratio in a fraction of the time, **Zstandard** was chosen as the algorithm to be implemented within the Data Delivery System. ----- +--- # 4. In-transit vs local encryption: Local (for now, looking at options) -__Date/year:__ 2019/2020 + +**Date/year:** 2019/2020 The most efficient way of delivering the data to the owners would be to perform encryption- and decryption in-transit, thereby decreasing the amount of memory required locally and possibly the total delivery time. However, there have been some difficulties finding a working solution for this, including that the `crypt4gh` Python package used in the beginning of the development did not support it. @@ -104,10 +109,11 @@ On further investigation and contact with Safespring, we learned: - All users of the Safespring backup service perform encryption on their own and handle the keys themselves. - Due to this, the encryption will be performed locally before upload to the S3 storage. ------ +--- # 5. Encryption Algorithm: ChaCha20-Poly1305 -__Date/year:__ 2019/2020 + +**Date/year:** 2019/2020 The new encryption format standard for genomics and health related data is Crypt4GH, developed by the Global Alliance for Genomics and Health and first released in 2019. The general encryption standard, however, is AES of which AES-GCM is an authenticated encryption mode. AES is currently (since 2001) the block cipher Rijndael. ChaCha20 is a stream cipher and is used within the Crypt4GH format. The most secure, efficient and generally appropriate format and algorithm should be implemented within the Data Delivery System. @@ -217,10 +223,11 @@ Files larger than 256 GiB will need to be partitioned, however the number of par Due to this, **ChaCha20-Poly1305** was chosen as the encryption algorithm within the Data Delivery System. ----- +--- # 6. File chunk size: 64 KiB -__Date/year:__ 2019/2020 + +**Date/year:** 2019/2020 Compression and encryption is performed in a streamed manner to avoid reading large files completely in memory. This is done by reading the file in chunks, and the size of the chunks affect the speed of the algorithms, their memory usage, and the final size of the compressed and encrypted files. @@ -232,10 +239,11 @@ To find the optimal chunk size, a 33 GB file was compressed and encrypted using The Data Delivery System will read the files in 64 KiB chunks. ------ +--- # 7. File integrity guarantee: Nonce incrementation -__Date/year:__ 2019/2020 + +**Date/year:** 2019/2020 As described in section 5. above, the Crypt4GH format encrypts the files in blocks of 64 KiB, after which each data blocks unique nonce, ciphertext and MAC are saved to the c4gh file. This guarantees the integrity of the data blocks, however it does not guarantee the integrity of the entire file, and it is therefore possible that some blocks are rearranged, duplicated or missing, without the recipient knowing. Although we have chosen to not use the Crypt4GH format within the delivery system, we do use the same encryption algorithm – ChaCha20-Poly1305 – and (since we cannot read huge files in memory) we have chosen to read the files in equally sized chunks. Therefore the integrity issue can potentially give huge problems for the delivery system. @@ -263,10 +271,11 @@ Due to this, no checksum verification is used during the upload. However, the fi Nonce incrementation will be used and no checksum verification will be performed during upload. ------ +--- # 8. Password Authentication: Argon2id -__Date/year:__ 2021 + +**Date/year:** 2021 Argon2 is also available in two other versions. These are argon2d (strong GPU resistance) and argon2i (resistant to side-channel attacks). Argon2id is a combination of the two and is the recommended mode. @@ -276,78 +285,92 @@ The Data Delivery System will use [Argon2id](https://github.com/hynek/argon2-cff > The chosen parameters will be added here soon. ----- +--- # 9. User roles: Super Admin, Unit Admin, Unit Personnel, Researcher -__Date:__ September 14th 2021 + +**Date:** September 14th 2021 ## Decision ### Super Admin (DC) + - Manage: Add, Remove, Edit - - Unit (instances) - - Users + - Unit (instances) + - Users ### Unit Admin + - Unit Personnel Permissions - Manage: Add, Add to project, Remove from project, Remove account, Change permissions - - Unit Admin - - Unit User + - Unit Admin + - Unit User ### Unit Personnel + - Project Owner Permissons - Upload - Delete ### Project Owner + - Research User Permissions - Manage: Invite, Add to project, Remove from project, Remove account, Change permissions - - Project Owners - - Research Users + - Project Owners + - Research Users ### Research Account + - Remove own account - List - Download ----- +--- + # 10. HOTP as default -__Date:__ December 1st 2022 -Initially, TOTP was implemented as the Two Factor Authentication. Authentication apps such as Authy or Google Authenticator could be set up and used to identify a user. However, due to some technical difficulties for some users, it was decided that we need to allow 2FA via mail as a default. +**Date:** December 1st 2022 + +Initially, TOTP was implemented as the Two Factor Authentication. Authentication apps such as Authy or Google Authenticator could be set up and used to identify a user. However, due to some technical difficulties for some users, it was decided that we need to allow 2FA via mail as a default. ## Decision -Use email 2FA (using HOTP) as a default. 2FA with authenticator apps (with TOTP) will be implemented _at some point_ and the users will be able to choose which method they want to use. ----- +Use email 2FA (using HOTP) as a default. 2FA with authenticator apps (with TOTP) will be implemented _at some point_ and the users will be able to choose which method they want to use. + +--- + # 11. Structured logging for action log -__Date:__ January 12th 2022 + +**Date:** January 12th 2022 Example: https://newrelic.com/blog/how-to-relic/python-structured-logging ## Decision -Structured logging should be implemented on the action logging first - the logging which saves when a user tries to perform a specific action e.g. upload/download/list/auth/rm etc. + +Structured logging should be implemented on the action logging first - the logging which saves when a user tries to perform a specific action e.g. upload/download/list/auth/rm etc. The information required to be logged: username, action, result (failed/successful), time, project in which the action was attempted. When the action logs have been fixed we will discuss whether or not this will be implemented in the general logging as well, such as debugging and general system info. - ----- +--- # 12. Requirements: No pinned versions -__Date:__ March 1st 2022 + +**Date:** March 1st 2022 ## Decisions We will not pin the requirement versions. If at some point something stops working we will look into it then and update the requirements then. This will simplify the installation for the users which is one of our priorities. ----- +--- # 13. No `--username` option -__Date:__ March 2nd 2022 -Previously there was a `--username` option for all commands where the user could specify the username. +**Date:** March 2nd 2022 + +Previously there was a `--username` option for all commands where the user could specify the username. ## Decision -We will not have the `--username` option. When using `dds auth login` command, either the existing encrypted token will be used or the user will be prompted to fill in the username and password. + +We will not have the `--username` option. When using `dds auth login` command, either the existing encrypted token will be used or the user will be prompted to fill in the username and password. From c602398372d0b5b3ad1a2b2fd704a65be11aad0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Mon, 21 Mar 2022 19:31:51 +0100 Subject: [PATCH 176/293] restrict super admin access and make sure super admin does not delete own account, and unit admin can only delete own account if num admins are more than 3 --- dds_web/api/files.py | 18 +++++++++--------- dds_web/api/project.py | 8 ++++---- dds_web/api/s3.py | 2 +- dds_web/api/user.py | 17 +++++++++++++++-- 4 files changed, 29 insertions(+), 16 deletions(-) diff --git a/dds_web/api/files.py b/dds_web/api/files.py index 20a307c2e..2e103f357 100644 --- a/dds_web/api/files.py +++ b/dds_web/api/files.py @@ -74,7 +74,7 @@ def check_eligibility_for_deletion(status, has_been_available): class NewFile(flask_restful.Resource): """Inserts a file into the database""" - @auth.login_required(role=["Super Admin", "Unit Admin", "Unit Personnel"]) + @auth.login_required(role=["Unit Admin", "Unit Personnel"]) @logging_bind_request @json_required @handle_validation_errors @@ -100,7 +100,7 @@ def post(self): return {"message": f"File '{new_file.name}' added to db."} - @auth.login_required(role=["Super Admin", "Unit Admin", "Unit Personnel"]) + @auth.login_required(role=["Unit Admin", "Unit Personnel"]) @logging_bind_request @handle_validation_errors def put(self): @@ -186,7 +186,7 @@ def put(self): class MatchFiles(flask_restful.Resource): """Checks for matching files in database""" - @auth.login_required(role=["Super Admin", "Unit Admin", "Unit Personnel"]) + @auth.login_required(role=["Unit Admin", "Unit Personnel"]) @logging_bind_request @json_required @handle_validation_errors @@ -218,7 +218,7 @@ def get(self): class ListFiles(flask_restful.Resource): """Lists files within a project""" - @auth.login_required + @auth.login_required(role=["Unit Admin", "Unit Personnel", "Project Owner", "Researcher"]) @logging_bind_request @handle_validation_errors def get(self): @@ -363,7 +363,7 @@ def items_in_subpath(project, folder="."): class RemoveFile(flask_restful.Resource): """Removes files from the database and s3 with boto3.""" - @auth.login_required(role=["Super Admin", "Unit Admin", "Unit Personnel"]) + @auth.login_required(role=["Unit Admin", "Unit Personnel"]) @logging_bind_request @json_required @handle_validation_errors @@ -458,7 +458,7 @@ def delete_one(self, project, filename): class RemoveDir(flask_restful.Resource): """Removes one or more full directories from the database and s3.""" - @auth.login_required(role=["Super Admin", "Unit Admin", "Unit Personnel"]) + @auth.login_required(role=["Unit Admin", "Unit Personnel"]) @logging_bind_request @json_required @handle_validation_errors @@ -563,7 +563,7 @@ def queue_file_entry_deletion(self, files: list): class FileInfo(flask_restful.Resource): """Get file info on files to download.""" - @auth.login_required + @auth.login_required(role=["Unit Admin", "Unit Personnel", "Project Owner", "Researcher"]) @logging_bind_request @json_required @handle_validation_errors @@ -597,7 +597,7 @@ def get(self): class FileInfoAll(flask_restful.Resource): """Get info on all project files.""" - @auth.login_required + @auth.login_required(role=["Unit Admin", "Unit Personnel", "Project Owner", "Researcher"]) @logging_bind_request @handle_validation_errors def get(self): @@ -619,7 +619,7 @@ def get(self): class UpdateFile(flask_restful.Resource): """Update file info after download""" - @auth.login_required + @auth.login_required(role=["Unit Admin", "Unit Personnel", "Project Owner", "Researcher"]) @logging_bind_request @json_required @handle_validation_errors diff --git a/dds_web/api/project.py b/dds_web/api/project.py index f5ab60c1f..422e0ecf7 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -73,7 +73,7 @@ def get(self): return return_info - @auth.login_required(role=["Super Admin", "Unit Admin", "Unit Personnel"]) + @auth.login_required(role=["Unit Admin", "Unit Personnel"]) @logging_bind_request @json_required @handle_validation_errors @@ -348,7 +348,7 @@ def delete_project_info(self, proj): class GetPublic(flask_restful.Resource): """Gets the public key beloning to the current project.""" - @auth.login_required + @auth.login_required(role=["Unit Admin", "Unit Personnel", "Project Owner", "Researcher"]) @logging_bind_request @handle_validation_errors def get(self): @@ -367,7 +367,7 @@ def get(self): class GetPrivate(flask_restful.Resource): """Gets the private key belonging to the current project.""" - @auth.login_required + @auth.login_required(role=["Unit Admin", "Unit Personnel", "Project Owner", "Researcher"]) @logging_bind_request @handle_validation_errors def get(self): @@ -486,7 +486,7 @@ def project_usage(project): class RemoveContents(flask_restful.Resource): """Removes all project contents.""" - @auth.login_required(role=["Super Admin", "Unit Admin", "Unit Personnel"]) + @auth.login_required(role=["Unit Admin", "Unit Personnel"]) @logging_bind_request @dbsession @handle_validation_errors diff --git a/dds_web/api/s3.py b/dds_web/api/s3.py index 329234ef6..0c5c7486c 100644 --- a/dds_web/api/s3.py +++ b/dds_web/api/s3.py @@ -30,7 +30,7 @@ class S3Info(flask_restful.Resource): """Gets the projects S3 keys""" - @auth.login_required + @auth.login_required(role=["Unit Admin", "Unit Personnel"]) @logging_bind_request @handle_validation_errors def get(self): diff --git a/dds_web/api/user.py b/dds_web/api/user.py index af72b590c..37b048c65 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -494,7 +494,7 @@ class DeleteUserSelf(flask_restful.Resource): Every user can self-delete the own account with an e-mail confirmation. """ - @auth.login_required + @auth.login_required(role=["Unit Admin", "Unit Personnel", "Project Owner", "Researcher"]) @logging_bind_request def delete(self): """Request deletion of own account.""" @@ -508,6 +508,19 @@ def delete(self): if current_user.role != "Super Admin": proj_ids = [proj.public_id for proj in current_user.projects] + if current_user.role == "Unit Admin": + num_admins = models.UnitUser.query.filter_by( + unit_id=current_user.unit.id, is_admin=True + ).count() + if num_admins <= 3: + raise ddserr.AccessDeniedError( + message=( + f"Your unit only has {num_admins} Unit Admins. " + "You cannot delete your account. " + "Invite a new Unit Admin first if you wish to proceed." + ) + ) + # Create URL safe token for invitation link s = itsdangerous.URLSafeTimedSerializer(flask.current_app.config["SECRET_KEY"]) token = s.dumps(email_str, salt="email-delete") @@ -764,7 +777,7 @@ def delete_user(user): class RemoveUserAssociation(flask_restful.Resource): - @auth.login_required + @auth.login_required(role=["Unit Admin", "Unit Personnel", "Project Owner", "Researcher"]) @logging_bind_request @json_required @handle_validation_errors From a7700c4351c135f181af6fc778305562e3465ee3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Tue, 22 Mar 2022 12:57:15 +0100 Subject: [PATCH 177/293] add check to delete user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Ina Odén Österbo --- dds_web/api/user.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index af72b590c..a17159e6a 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -700,8 +700,18 @@ class DeleteUser(flask_restful.Resource): @logging_bind_request @handle_validation_errors def delete(self): - - user = user_schemas.UserSchema().load(flask.request.json) + """Delete user or invite in the DDS.""" + json_info = flask.request.json + if json_info: + is_invite = json_info.pop("is_invite", False) + if is_invite: + unanswered_invite = user_schemas.UnansweredInvite().load(json_info) + if unanswered_invite: + db.session.delete(unanswered_invite) + db.session.commit() + return {"message": "The invite has been deleted."} + + user = user_schemas.UserSchema().load(json_info) if not user: raise ddserr.UserDeletionError( message=( From 53dedb2283e7ff19ecb35ee6d66379f2e1f7da65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Tue, 22 Mar 2022 13:22:10 +0100 Subject: [PATCH 178/293] add function for invite deletion --- dds_web/api/user.py | 40 +++++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index a17159e6a..0bb07c01f 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -701,15 +701,16 @@ class DeleteUser(flask_restful.Resource): @handle_validation_errors def delete(self): """Delete user or invite in the DDS.""" + current_user = auth.current_user() + json_info = flask.request.json if json_info: is_invite = json_info.pop("is_invite", False) if is_invite: - unanswered_invite = user_schemas.UnansweredInvite().load(json_info) - if unanswered_invite: - db.session.delete(unanswered_invite) - db.session.commit() - return {"message": "The invite has been deleted."} + email = self.delete_invite(email=json_info.get("email")) + return { + "message": ("The invite connected to email " f"'{email}' has been deleted.") + } user = user_schemas.UserSchema().load(json_info) if not user: @@ -772,6 +773,35 @@ def delete_user(user): db.session.rollback() raise ddserr.DatabaseError(message=str(err)) + @staticmethod + def delete_invite(email): + current_user_role = auth.current_user().role + try: + unanswered_invite = user_schemas.UnansweredInvite().load({"email": email}) + if unanswered_invite: + if ( + current_user_role == "Super Admin" + and unanswered_invite.role not in ["Super Admin", "Unit Admin"] + ) or (current_user_role == "Unit Admin" and unanswered_invite.role != "Unit Admin"): + raise ddserr.AccessDeniedError( + "You do not have the correct permissions to delete this invite." + ) + db.session.delete(unanswered_invite) + db.session.commit() + except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err: + db.session.rollback() + flask.current_app.logger.error( + "The invite connected to the email " + f"{email or '[no email provided]'} was not deleted." + ) + if isinstance(err, sqlalchemy.exc.OperationalError): + err_msg = "Database malfunction." + else: + err_msg = str(err) + raise ddserr.DatabaseError(message=err_msg) from err + + return unanswered_invite.email + class RemoveUserAssociation(flask_restful.Resource): @auth.login_required From 7e037fbb464113263053e2a2c31b199903daef3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Tue, 22 Mar 2022 13:26:57 +0100 Subject: [PATCH 179/293] super admins can remove all invites, unit admins every one except super admins --- dds_web/api/user.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index 0bb07c01f..acf21a9f6 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -779,10 +779,9 @@ def delete_invite(email): try: unanswered_invite = user_schemas.UnansweredInvite().load({"email": email}) if unanswered_invite: - if ( - current_user_role == "Super Admin" - and unanswered_invite.role not in ["Super Admin", "Unit Admin"] - ) or (current_user_role == "Unit Admin" and unanswered_invite.role != "Unit Admin"): + if not current_user_role == "Super Admin" or ( + current_user_role == "Unit Admin" and unanswered_invite.role == "Super Admin" + ): raise ddserr.AccessDeniedError( "You do not have the correct permissions to delete this invite." ) From aea5b954ae079149bca84ed1f591b8899a2fed56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Tue, 22 Mar 2022 14:02:21 +0100 Subject: [PATCH 180/293] tests --- dds_web/api/user.py | 14 +++-- tests/test_user_delete.py | 121 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 6 deletions(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index acf21a9f6..8517fc5bd 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -779,14 +779,16 @@ def delete_invite(email): try: unanswered_invite = user_schemas.UnansweredInvite().load({"email": email}) if unanswered_invite: - if not current_user_role == "Super Admin" or ( - current_user_role == "Unit Admin" and unanswered_invite.role == "Super Admin" + if current_user_role == "Super Admin" or ( + current_user_role == "Unit Admin" + and unanswered_invite.role in ["Unit Admin", "Unit Personnel", "Researcher"] ): + db.session.delete(unanswered_invite) + db.session.commit() + else: raise ddserr.AccessDeniedError( - "You do not have the correct permissions to delete this invite." + message="You do not have the correct permissions to delete this invite." ) - db.session.delete(unanswered_invite) - db.session.commit() except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err: db.session.rollback() flask.current_app.logger.error( @@ -799,7 +801,7 @@ def delete_invite(email): err_msg = str(err) raise ddserr.DatabaseError(message=err_msg) from err - return unanswered_invite.email + return email class RemoveUserAssociation(flask_restful.Resource): diff --git a/tests/test_user_delete.py b/tests/test_user_delete.py index fa7e35ba2..300638531 100644 --- a/tests/test_user_delete.py +++ b/tests/test_user_delete.py @@ -286,3 +286,124 @@ def test_del_request_others_superaction(client): exists = user_from_email(email_to_delete) assert exists is None assert dds_web.utils.email_in_db(email_to_delete) is False + + +# Test delete invite +def test_del_invite_non_existent(client): + """Super admin deletes non existent invite.""" + email_to_delete = "incorrect@mailtrap.io" + response = client.delete( + tests.DDSEndpoint.USER_DELETE, + headers=tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client), + json={"email": email_to_delete, "is_invite": True}, + ) + assert response.status_code == http.HTTPStatus.OK + + assert ( + f"The invite connected to email '{email_to_delete}' has been deleted." + in response.json.get("message") + ) + + +def test_del_invite_no_email(client): + """Super admin deletes invite without specifying email.""" + response = client.delete( + tests.DDSEndpoint.USER_DELETE, + headers=tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client), + json={"is_invite": True}, + ) + assert response.status_code == http.HTTPStatus.BAD_REQUEST + assert response.json.get("email").get("message") == "The email cannot be null." + + +def test_del_invite_superadmin_as_superadmin(client): + """Super Admin invites super admin and deletes user.""" + invited_user = {"email": "test_user@mailtrap.io", "role": "Super Admin"} + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client), + json=invited_user, + ) + assert response.status_code == http.HTTPStatus.OK + invited_user_row = models.Invite.query.filter_by(email=invited_user["email"]).one_or_none() + assert invited_user_row and invited_user_row.role == "Super Admin" + + response = client.delete( + tests.DDSEndpoint.USER_DELETE, + headers=tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client), + json={"email": invited_user_row.email, "is_invite": True}, + ) + assert response.status_code == http.HTTPStatus.OK + assert ( + response.json.get("message") + == f"The invite connected to email '{invited_user_row.email}' has been deleted." + ) + + invited_user_row = models.Invite.query.filter_by(email=invited_user["email"]).one_or_none() + assert not invited_user_row + + +def test_del_invite_superadmin_as_unit_admin(client): + invited_user = {"email": "test_user@mailtrap.io", "role": "Super Admin"} + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client), + json=invited_user, + ) + assert response.status_code == http.HTTPStatus.OK + invited_user_row = models.Invite.query.filter_by(email=invited_user["email"]).one_or_none() + assert invited_user_row and invited_user_row.role == "Super Admin" + + response = client.delete( + tests.DDSEndpoint.USER_DELETE, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), + json={"email": invited_user_row.email, "is_invite": True}, + ) + assert response.status_code == http.HTTPStatus.FORBIDDEN + assert ( + response.json.get("message") + == "You do not have the correct permissions to delete this invite." + ) + + invited_user_row = models.Invite.query.filter_by(email=invited_user["email"]).one_or_none() + assert invited_user_row and invited_user_row.role == "Super Admin" + + +def test_del_invite_unit_admin_and_personnel_and_researcher_as_unit_admin(client): + unit = models.Unit.query.filter_by(name="Unit 1").one_or_none() + invited_users = [ + {"email": "unitadmin_invite@mailtrap.io", "role": "Unit Admin", "unit": unit.public_id}, + { + "email": "unitpersonnel_invite@mailtrap.io", + "role": "Unit Personnel", + "unit": unit.public_id, + }, + {"email": "researcher_invite@mailtrap.io", "role": "Researcher"}, + ] + for user in invited_users: + print(f"Inviting user: {user}") + response = client.post( + tests.DDSEndpoint.USER_ADD, + headers=tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client), + json=user, + ) + assert response.status_code == http.HTTPStatus.OK + invited_user_row = models.Invite.query.filter_by(email=user["email"]).one_or_none() + assert invited_user_row and invited_user_row.role == user["role"] + + print( + f"Deleting invited user: {invited_user_row.email} ({invited_user_row.role}) as {tests}" + ) + response = client.delete( + tests.DDSEndpoint.USER_DELETE, + headers=tests.UserAuth(tests.USER_CREDENTIALS["unitadmin"]).token(client), + json={"email": invited_user_row.email, "is_invite": True}, + ) + assert response.status_code == http.HTTPStatus.OK + assert ( + response.json.get("message") + == f"The invite connected to email '{invited_user_row.email}' has been deleted." + ) + + invited_user_row = models.Invite.query.filter_by(email=user["email"]).one_or_none() + assert not invited_user_row From 1136e19ef6c27f06abcb490680765b76f7bf71e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Tue, 22 Mar 2022 14:52:25 +0100 Subject: [PATCH 181/293] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a032b59af..972a88559 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,3 +56,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - Return int instead of string from files listing and only return usage info if right role ([#1070](https://github.com/ScilifelabDataCentre/dds_web/pull/1070)) - Batch deletion of files (breaking atomicity) ([#1067](https://github.com/ScilifelabDataCentre/dds_web/pull/1067)) - Change token expiration time to 7 days (168 hours) ([#1061](https://github.com/ScilifelabDataCentre/dds_web/pull/1061)) +- Add possibility of deleting invites (temporary fix in delete user endpoint) ([#1075](https://github.com/ScilifelabDataCentre/dds_web/pull/1075)) From c7b357fe5babb87715afe592c425a91c60f35ef7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Tue, 22 Mar 2022 15:44:03 +0100 Subject: [PATCH 182/293] create_new_unit --- dds_web/__init__.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/dds_web/__init__.py b/dds_web/__init__.py index 16690cbfc..2a81ec132 100644 --- a/dds_web/__init__.py +++ b/dds_web/__init__.py @@ -323,3 +323,33 @@ def fill_db_wrapper(db_type): dds_web.development.factories.create_all() flask.current_app.logger.info("DB filled") + + +@click.command("create-unit") +@click.option("name", type=str, required=True) +@click.option("public_id", type=str, required=True) +@click.option("external_display_name", type=str, required=True) +@click.option("contact_email", type=str, required=True) +@click.option("internal_ref", type=str, required=False) +@click.option("safespring_endpoint", type=str, required=True) +@click.option("safespring_name", type=str, required=True) +@click.option("safespring_access", type=str, required=True) +@click.option("safespring_secret", type=str, required=True) +@click.option("days_in_available", type=int, required=False, default=90) +@click.option("days_in_expired", type=int, required=False, default=30) +@flask.cli.with_appcontext +def create_new_unit( + name, + public_id, + external_display_name, + contact_email, + internal_ref, + safespring_endpoint, + safespring_name, + safespring_access, + safespring_secret, + days_in_available, + days_in_expired, +): + """Create a new unit.""" + from dds_web.database import models From 717deed1c79eecab03cfb59b0731f43ff28f1a35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Tue, 22 Mar 2022 15:47:57 +0100 Subject: [PATCH 183/293] added test for None --- tests/test_user_delete.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_user_delete.py b/tests/test_user_delete.py index 300638531..f36545497 100644 --- a/tests/test_user_delete.py +++ b/tests/test_user_delete.py @@ -316,6 +316,17 @@ def test_del_invite_no_email(client): assert response.json.get("email").get("message") == "The email cannot be null." +def test_del_invite_null_email(client): + """Super admin deletes invite without specifying email.""" + response = client.delete( + tests.DDSEndpoint.USER_DELETE, + headers=tests.UserAuth(tests.USER_CREDENTIALS["superadmin"]).token(client), + json={"email": None, "is_invite": True}, + ) + assert response.status_code == http.HTTPStatus.BAD_REQUEST + assert response.json.get("email").get("message") == "The email cannot be null." + + def test_del_invite_superadmin_as_superadmin(client): """Super Admin invites super admin and deletes user.""" invited_user = {"email": "test_user@mailtrap.io", "role": "Super Admin"} From 8bfffed0f40de77a9619f19d056daa9e3faf7d5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Tue, 22 Mar 2022 16:35:33 +0100 Subject: [PATCH 184/293] create new unit --- dds_web/__init__.py | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/dds_web/__init__.py b/dds_web/__init__.py index 2a81ec132..30e2a5e4c 100644 --- a/dds_web/__init__.py +++ b/dds_web/__init__.py @@ -246,6 +246,7 @@ def load_user(user_id): ) app.cli.add_command(fill_db_wrapper) + app.cli.add_command(create_new_unit) with app.app_context(): # Everything in here has access to sessions from dds_web.database import models @@ -326,17 +327,17 @@ def fill_db_wrapper(db_type): @click.command("create-unit") -@click.option("name", type=str, required=True) -@click.option("public_id", type=str, required=True) -@click.option("external_display_name", type=str, required=True) -@click.option("contact_email", type=str, required=True) -@click.option("internal_ref", type=str, required=False) -@click.option("safespring_endpoint", type=str, required=True) -@click.option("safespring_name", type=str, required=True) -@click.option("safespring_access", type=str, required=True) -@click.option("safespring_secret", type=str, required=True) -@click.option("days_in_available", type=int, required=False, default=90) -@click.option("days_in_expired", type=int, required=False, default=30) +@click.option("--name", "-n", type=str, required=True) +@click.option("--public_id", "-p", type=str, required=True) +@click.option("--external_display_name", "-e", type=str, required=True) +@click.option("--contact_email", "-c", type=str, required=True) +@click.option("--internal_ref", "-ref", type=str, required=False) +@click.option("--safespring_endpoint", "-se", type=str, required=True) +@click.option("--safespring_name", "-sn", type=str, required=True) +@click.option("--safespring_access", "-sa", type=str, required=True) +@click.option("--safespring_secret", "-ss", type=str, required=True) +@click.option("--days_in_available", "-da", type=int, required=False, default=90) +@click.option("--days_in_expired", "-de", type=int, required=False, default=30) @flask.cli.with_appcontext def create_new_unit( name, @@ -353,3 +354,21 @@ def create_new_unit( ): """Create a new unit.""" from dds_web.database import models + + new_unit = models.Unit( + name=name, + public_id=public_id, + external_display_name=external_display_name, + contact_email=contact_email, + internal_ref=internal_ref or public_id, + safespring_endpoint=safespring_endpoint, + safespring_name=safespring_name, + safespring_access=safespring_access, + safespring_secret=safespring_secret, + days_in_available=days_in_available, + days_in_expired=days_in_expired, + ) + db.session.add(new_unit) + db.session.commit() + + flask.current_app.logger.info(f"Unit '{name}' created") From ac664ae997698a0f351fd49371ddb64df9621496 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Tue, 22 Mar 2022 16:38:31 +0100 Subject: [PATCH 185/293] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a032b59af..53132ae5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,3 +56,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - Return int instead of string from files listing and only return usage info if right role ([#1070](https://github.com/ScilifelabDataCentre/dds_web/pull/1070)) - Batch deletion of files (breaking atomicity) ([#1067](https://github.com/ScilifelabDataCentre/dds_web/pull/1067)) - Change token expiration time to 7 days (168 hours) ([#1061](https://github.com/ScilifelabDataCentre/dds_web/pull/1061)) +- Flask command `create-unit` to create unit without having to interact with database directly () From bb25cfc3a4e880e81868ae717cf4a1c953846562 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Tue, 22 Mar 2022 17:38:22 +0100 Subject: [PATCH 186/293] email validation --- dds_web/api/schemas/user_schemas.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dds_web/api/schemas/user_schemas.py b/dds_web/api/schemas/user_schemas.py index f1f25fc52..707e34d81 100644 --- a/dds_web/api/schemas/user_schemas.py +++ b/dds_web/api/schemas/user_schemas.py @@ -31,6 +31,7 @@ class UserSchema(marshmallow.Schema): "required": {"message": "A user email is required."}, "null": {"message": "The user email cannot be null."}, }, + validate=marshmallow.validate.Email(), ) class Meta: @@ -57,6 +58,7 @@ class UnansweredInvite(marshmallow.Schema): "required": {"message": "An email is required."}, "null": {"message": "The email cannot be null."}, }, + validate=marshmallow.validate.Email(), ) class Meta: From 97d85d2dea1deddb8307f02cf42515498373f8c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Tue, 22 Mar 2022 18:08:45 +0100 Subject: [PATCH 187/293] catch operational error --- dds_web/api/dds_decorators.py | 3 +++ dds_web/api/project.py | 12 ++++++++++-- dds_web/api/schemas/user_schemas.py | 2 -- dds_web/api/user.py | 25 ++++++++++++++++++++----- dds_web/web/user.py | 3 +++ 5 files changed, 36 insertions(+), 9 deletions(-) diff --git a/dds_web/api/dds_decorators.py b/dds_web/api/dds_decorators.py index 0a7576c14..67bf3708a 100644 --- a/dds_web/api/dds_decorators.py +++ b/dds_web/api/dds_decorators.py @@ -14,6 +14,7 @@ import structlog import sqlalchemy import marshmallow +import pymysql # Own modules from dds_web import db @@ -84,6 +85,8 @@ def make_commit(*args, **kwargs): # Run function, catch errors try: result = func(*args, **kwargs) + except pymysql.err.OperationalError as err: + raise DatabaseError(message=str(err), alt_message="Unexpected database error.") except: db.session.rollback() raise diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 422e0ecf7..0703e0494 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -14,6 +14,7 @@ import datetime import botocore import marshmallow +import pymysql # Own modules import dds_web.utils @@ -609,8 +610,15 @@ def post(self): for user in p_info["users_to_add"]: try: existing_user = user_schemas.UserSchema().load(user) - except marshmallow.exceptions.ValidationError as err: - addition_status = f"Error for {user.get('email')}: {err}" + except ( + marshmallow.exceptions.ValidationError, + pymysql.err.OperationalError, + ) as err: + if isinstance(err, pymysql.err.OperationalError): + flask.current_app.logger.error(err) + addition_status = "Unexpected database error." + else: + addition_status = f"Error for {user.get('email')}: {err}" user_addition_statuses.append(addition_status) continue diff --git a/dds_web/api/schemas/user_schemas.py b/dds_web/api/schemas/user_schemas.py index 707e34d81..f1f25fc52 100644 --- a/dds_web/api/schemas/user_schemas.py +++ b/dds_web/api/schemas/user_schemas.py @@ -31,7 +31,6 @@ class UserSchema(marshmallow.Schema): "required": {"message": "A user email is required."}, "null": {"message": "The user email cannot be null."}, }, - validate=marshmallow.validate.Email(), ) class Meta: @@ -58,7 +57,6 @@ class UnansweredInvite(marshmallow.Schema): "required": {"message": "An email is required."}, "null": {"message": "The email cannot be null."}, }, - validate=marshmallow.validate.Email(), ) class Meta: diff --git a/dds_web/api/user.py b/dds_web/api/user.py index 37b048c65..92e83d828 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -18,6 +18,7 @@ import structlog import sqlalchemy import http +import pymysql # Own modules @@ -79,8 +80,11 @@ def post(self): send_email = json_info.get("send_email", True) # Check if email is registered to a user - existing_user = user_schemas.UserSchema().load({"email": email}) - unanswered_invite = user_schemas.UnansweredInvite().load({"email": email}) + try: + existing_user = user_schemas.UserSchema().load({"email": email}) + unanswered_invite = user_schemas.UnansweredInvite().load({"email": email}) + except pymysql.err.OperationalError as err: + raise ddserr.DatabaseError(message=str(err), alt_message="Unexpected database error.") if existing_user or unanswered_invite: if not project: @@ -619,7 +623,11 @@ def post(self): if "email" not in json_input: raise ddserr.DDSArgumentError(message="User email missing.") - user = user_schemas.UserSchema().load({"email": json_input.pop("email")}) + try: + user = user_schemas.UserSchema().load({"email": json_input.pop("email")}) + except pymysql.err.OperationalError as err: + raise ddserr.DatabaseError(message=str(err), alt_message="Unexpected database error.") + if not user: raise ddserr.NoSuchUserError() @@ -714,7 +722,11 @@ class DeleteUser(flask_restful.Resource): @handle_validation_errors def delete(self): - user = user_schemas.UserSchema().load(flask.request.json) + try: + user = user_schemas.UserSchema().load(flask.request.json) + except pymysql.err.OperationalError as err: + raise ddserr.DatabaseError(message=str(err), alt_message="Unexpected database error.") + if not user: raise ddserr.UserDeletionError( message=( @@ -790,7 +802,10 @@ def post(self): raise ddserr.DDSArgumentError(message="User email missing.") # Check if email is registered to a user - existing_user = user_schemas.UserSchema().load({"email": user_email}) + try: + existing_user = user_schemas.UserSchema().load({"email": user_email}) + except pymysql.err.OperationalError as err: + raise ddserr.DatabaseError(message=str(err), alt_message="Unexpected database error.") if not existing_user: raise ddserr.NoSuchUserError( diff --git a/dds_web/web/user.py b/dds_web/web/user.py index f8f68ee48..f744fe825 100644 --- a/dds_web/web/user.py +++ b/dds_web/web/user.py @@ -12,6 +12,7 @@ import flask_login import itsdangerous import sqlalchemy +import pymysql # Own Modules import dds_web.utils @@ -539,6 +540,8 @@ def confirm_self_deletion(token): message=f"User deletion request for {user.username} / {user.primary_email.email} failed due to database error: {sqlerr}", alt_message=f"Deletion request for user {user.username} registered with {user.primary_email.email} failed for technical reasons. Please contact the unit for technical support!", ) + except pymysql.err.OperationalError as err: + raise ddserr.DatabaseError(message=str(err), alt_message="Unexpected database error.") flask.session.clear() From 96d155a201eb70c58070d92ed74b04f200f32b24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Tue, 22 Mar 2022 18:15:46 +0100 Subject: [PATCH 188/293] add sqlalchemy errors too --- dds_web/api/dds_decorators.py | 2 +- dds_web/api/project.py | 5 ++++- dds_web/api/user.py | 10 ++++++---- dds_web/web/user.py | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/dds_web/api/dds_decorators.py b/dds_web/api/dds_decorators.py index 67bf3708a..f62b88289 100644 --- a/dds_web/api/dds_decorators.py +++ b/dds_web/api/dds_decorators.py @@ -85,7 +85,7 @@ def make_commit(*args, **kwargs): # Run function, catch errors try: result = func(*args, **kwargs) - except pymysql.err.OperationalError as err: + except (pymysql.err.OperationalError, sqlalchemy.exc.SQLAlchemyError) as err: raise DatabaseError(message=str(err), alt_message="Unexpected database error.") except: db.session.rollback() diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 0703e0494..b6f4ee094 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -613,8 +613,11 @@ def post(self): except ( marshmallow.exceptions.ValidationError, pymysql.err.OperationalError, + sqlalchemy.exc.SQLAlchemyError, ) as err: - if isinstance(err, pymysql.err.OperationalError): + if isinstance(err, pymysql.err.OperationalError) or isinstance( + err, sqlalchemy.exc.SQLAlchemyError + ): flask.current_app.logger.error(err) addition_status = "Unexpected database error." else: diff --git a/dds_web/api/user.py b/dds_web/api/user.py index 92e83d828..66389364b 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -80,12 +80,14 @@ def post(self): send_email = json_info.get("send_email", True) # Check if email is registered to a user + flask.current_app.logger.debug("Here") try: existing_user = user_schemas.UserSchema().load({"email": email}) unanswered_invite = user_schemas.UnansweredInvite().load({"email": email}) - except pymysql.err.OperationalError as err: + except (pymysql.err.OperationalError, sqlalchemy.exc.SQLAlchemyError) as err: raise ddserr.DatabaseError(message=str(err), alt_message="Unexpected database error.") + flask.current_app.logger.debug("Here") if existing_user or unanswered_invite: if not project: raise ddserr.DDSArgumentError( @@ -625,7 +627,7 @@ def post(self): try: user = user_schemas.UserSchema().load({"email": json_input.pop("email")}) - except pymysql.err.OperationalError as err: + except (pymysql.err.OperationalError, sqlalchemy.exc.SQLAlchemyError) as err: raise ddserr.DatabaseError(message=str(err), alt_message="Unexpected database error.") if not user: @@ -724,7 +726,7 @@ def delete(self): try: user = user_schemas.UserSchema().load(flask.request.json) - except pymysql.err.OperationalError as err: + except (pymysql.err.OperationalError, sqlalchemy.exc.SQLAlchemyError) as err: raise ddserr.DatabaseError(message=str(err), alt_message="Unexpected database error.") if not user: @@ -804,7 +806,7 @@ def post(self): # Check if email is registered to a user try: existing_user = user_schemas.UserSchema().load({"email": user_email}) - except pymysql.err.OperationalError as err: + except (pymysql.err.OperationalError, sqlalchemy.exc.SQLAlchemyError) as err: raise ddserr.DatabaseError(message=str(err), alt_message="Unexpected database error.") if not existing_user: diff --git a/dds_web/web/user.py b/dds_web/web/user.py index f744fe825..e0f0f05b3 100644 --- a/dds_web/web/user.py +++ b/dds_web/web/user.py @@ -540,7 +540,7 @@ def confirm_self_deletion(token): message=f"User deletion request for {user.username} / {user.primary_email.email} failed due to database error: {sqlerr}", alt_message=f"Deletion request for user {user.username} registered with {user.primary_email.email} failed for technical reasons. Please contact the unit for technical support!", ) - except pymysql.err.OperationalError as err: + except (pymysql.err.OperationalError, sqlalchemy.exc.SQLAlchemyError) as err: raise ddserr.DatabaseError(message=str(err), alt_message="Unexpected database error.") flask.session.clear() From 705a6b181e99240b86f8e78c9badc4cf22534bf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Tue, 22 Mar 2022 18:17:40 +0100 Subject: [PATCH 189/293] Update dds_web/api/user.py --- dds_web/api/user.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index 66389364b..bbe678f6d 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -80,7 +80,6 @@ def post(self): send_email = json_info.get("send_email", True) # Check if email is registered to a user - flask.current_app.logger.debug("Here") try: existing_user = user_schemas.UserSchema().load({"email": email}) unanswered_invite = user_schemas.UnansweredInvite().load({"email": email}) From 72ccfc974e1b51626de9c5dc5266efda4076ebe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Tue, 22 Mar 2022 18:17:53 +0100 Subject: [PATCH 190/293] Update dds_web/api/user.py --- dds_web/api/user.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index bbe678f6d..a4dc99f5c 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -86,7 +86,6 @@ def post(self): except (pymysql.err.OperationalError, sqlalchemy.exc.SQLAlchemyError) as err: raise ddserr.DatabaseError(message=str(err), alt_message="Unexpected database error.") - flask.current_app.logger.debug("Here") if existing_user or unanswered_invite: if not project: raise ddserr.DDSArgumentError( From 5d8af9e4a23b5048d7e1d9ff894dc0eb53d63f87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Wed, 23 Mar 2022 09:12:41 +0100 Subject: [PATCH 191/293] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Linus Östberg --- dds_web/api/dds_decorators.py | 3 +-- dds_web/api/project.py | 8 ++------ dds_web/api/user.py | 9 ++++----- dds_web/web/user.py | 3 +-- 4 files changed, 8 insertions(+), 15 deletions(-) diff --git a/dds_web/api/dds_decorators.py b/dds_web/api/dds_decorators.py index f62b88289..57e5925af 100644 --- a/dds_web/api/dds_decorators.py +++ b/dds_web/api/dds_decorators.py @@ -14,7 +14,6 @@ import structlog import sqlalchemy import marshmallow -import pymysql # Own modules from dds_web import db @@ -85,7 +84,7 @@ def make_commit(*args, **kwargs): # Run function, catch errors try: result = func(*args, **kwargs) - except (pymysql.err.OperationalError, sqlalchemy.exc.SQLAlchemyError) as err: + except sqlalchemy.exc.OperationalError as err: raise DatabaseError(message=str(err), alt_message="Unexpected database error.") except: db.session.rollback() diff --git a/dds_web/api/project.py b/dds_web/api/project.py index b6f4ee094..b5bb729e5 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -14,7 +14,6 @@ import datetime import botocore import marshmallow -import pymysql # Own modules import dds_web.utils @@ -612,12 +611,9 @@ def post(self): existing_user = user_schemas.UserSchema().load(user) except ( marshmallow.exceptions.ValidationError, - pymysql.err.OperationalError, - sqlalchemy.exc.SQLAlchemyError, + sqlalchemy.exc.OperationalError, ) as err: - if isinstance(err, pymysql.err.OperationalError) or isinstance( - err, sqlalchemy.exc.SQLAlchemyError - ): + if isinstance(err, sqlalchemy.exc.OperationalError): flask.current_app.logger.error(err) addition_status = "Unexpected database error." else: diff --git a/dds_web/api/user.py b/dds_web/api/user.py index a4dc99f5c..6f7431b74 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -18,7 +18,6 @@ import structlog import sqlalchemy import http -import pymysql # Own modules @@ -83,7 +82,7 @@ def post(self): try: existing_user = user_schemas.UserSchema().load({"email": email}) unanswered_invite = user_schemas.UnansweredInvite().load({"email": email}) - except (pymysql.err.OperationalError, sqlalchemy.exc.SQLAlchemyError) as err: + except sqlalchemy.exc.OperationalError as err: raise ddserr.DatabaseError(message=str(err), alt_message="Unexpected database error.") if existing_user or unanswered_invite: @@ -625,7 +624,7 @@ def post(self): try: user = user_schemas.UserSchema().load({"email": json_input.pop("email")}) - except (pymysql.err.OperationalError, sqlalchemy.exc.SQLAlchemyError) as err: + except sqlalchemy.exc.OperationalError as err: raise ddserr.DatabaseError(message=str(err), alt_message="Unexpected database error.") if not user: @@ -724,7 +723,7 @@ def delete(self): try: user = user_schemas.UserSchema().load(flask.request.json) - except (pymysql.err.OperationalError, sqlalchemy.exc.SQLAlchemyError) as err: + except sqlalchemy.exc.OperationalError as err: raise ddserr.DatabaseError(message=str(err), alt_message="Unexpected database error.") if not user: @@ -804,7 +803,7 @@ def post(self): # Check if email is registered to a user try: existing_user = user_schemas.UserSchema().load({"email": user_email}) - except (pymysql.err.OperationalError, sqlalchemy.exc.SQLAlchemyError) as err: + except sqlalchemy.exc.OperationalError as err: raise ddserr.DatabaseError(message=str(err), alt_message="Unexpected database error.") if not existing_user: diff --git a/dds_web/web/user.py b/dds_web/web/user.py index e0f0f05b3..7cc97dd32 100644 --- a/dds_web/web/user.py +++ b/dds_web/web/user.py @@ -12,7 +12,6 @@ import flask_login import itsdangerous import sqlalchemy -import pymysql # Own Modules import dds_web.utils @@ -540,7 +539,7 @@ def confirm_self_deletion(token): message=f"User deletion request for {user.username} / {user.primary_email.email} failed due to database error: {sqlerr}", alt_message=f"Deletion request for user {user.username} registered with {user.primary_email.email} failed for technical reasons. Please contact the unit for technical support!", ) - except (pymysql.err.OperationalError, sqlalchemy.exc.SQLAlchemyError) as err: + except sqlalchemy.exc.OperationalError as err: raise ddserr.DatabaseError(message=str(err), alt_message="Unexpected database error.") flask.session.clear() From fcc0007566d2f37784beb53b98a6d4b9d975dcee Mon Sep 17 00:00:00 2001 From: Anandashankar Anil Date: Wed, 23 Mar 2022 10:50:11 +0100 Subject: [PATCH 192/293] Add flask command for uploading file upload error log --- dds_web/__init__.py | 66 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/dds_web/__init__.py b/dds_web/__init__.py index 30e2a5e4c..ecc534d85 100644 --- a/dds_web/__init__.py +++ b/dds_web/__init__.py @@ -247,6 +247,7 @@ def load_user(user_id): app.cli.add_command(fill_db_wrapper) app.cli.add_command(create_new_unit) + app.cli.add_command(update_uploaded_file_with_log) with app.app_context(): # Everything in here has access to sessions from dds_web.database import models @@ -372,3 +373,68 @@ def create_new_unit( db.session.commit() flask.current_app.logger.info(f"Unit '{name}' created") + + +@click.command("update-uploaded-file") +@click.option("--project", "-p", type=str, required=True) +@click.option("--path-to-log-file", "-fp", type=str, required=True) +@flask.cli.with_appcontext +def update_uploaded_file_with_log(project, path_to_log_file): + """Update file details that weren't properly uploaded to db from cli log""" + import botocore + from dds_web.database import models + from dds_web.api.project_schemas import verify_project_exists + from dds_web import db + from dds_web.api.api_s3_connector import ApiS3Connector + from dds_web.api.schemas import file_schemas + + proj_in_db = verify_project_exists(project) + with open(path_to_log_file, "r") as f: + log = json.load(f) + errors = {} + files_added = [] + for file in log: + with ApiS3Connector(project=proj_in_db) as s3conn: + try: + _ = s3conn.resource.meta.client.head_object( + Bucket=s3conn.project.bucket, Key=log["path_remote"] + ) + except botocore.client.ClientError as err: + if err.response["Error"]["Code"] == "404": + errors[file] = {"error": "File not found in S3", "traceback": err.__traceback__} + else: + file = models.File.query.filter( + sqlalchemy.and_( + models.File.name == sqlalchemy.func.binary(file), + models.File.project_id == proj_in_db.id, + ) + ).first() + if file: + file.name_in_bucket = log["path_remote"] + file.subpath = log["subpath"] + file.project = proj_in_db.public_id + file.size_original = log["size_raw"] + file.size_stored = log["size_processed"] + file.compressed = log["compressed"] + file.public_key = log["public_key"] + file.salt = log["salt"] + file.checksum = log["checksum"] + else: + new_file = file_schemas.NewFileSchema().load( + { + "name": file, + "name_in_bucket": log["path_remote"], + "subpath": log["subpath"], + "project": proj_in_db.public_id, + "size_original": log["size_raw"], + "size_stored": log["size_processed"], + "compressed": log["compressed"], + "public_key": log["public_key"], + "salt": log["salt"], + "checksum": log["checksum"], + } + ) + files_added.append(file) + db.session.commit() + flask.current_app.logger.info(f"Files added: {files_added}") + flask.current_app.logger.info(f"Errors while adding files: {errors}") From c24a5052739dbf42c1e6737e4cce72adf1c11317 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Wed, 23 Mar 2022 12:58:13 +0100 Subject: [PATCH 193/293] flask command --- dds_web/__init__.py | 56 +++++++++++++++++++++----------------------- dds_web/api/files.py | 1 + 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/dds_web/__init__.py b/dds_web/__init__.py index ecc534d85..ccf0a6fc2 100644 --- a/dds_web/__init__.py +++ b/dds_web/__init__.py @@ -383,22 +383,28 @@ def update_uploaded_file_with_log(project, path_to_log_file): """Update file details that weren't properly uploaded to db from cli log""" import botocore from dds_web.database import models - from dds_web.api.project_schemas import verify_project_exists + from dds_web.api.schemas.project_schemas import verify_project_exists from dds_web import db from dds_web.api.api_s3_connector import ApiS3Connector from dds_web.api.schemas import file_schemas + import json + + proj_in_db = models.Project.query.filter_by(public_id=project).one_or_none() + assert proj_in_db - proj_in_db = verify_project_exists(project) with open(path_to_log_file, "r") as f: log = json.load(f) errors = {} files_added = [] - for file in log: + for file, vals in log.items(): + status = vals.get("status") + if not status or not status.get("failed_op") == "add_file_db": + continue + with ApiS3Connector(project=proj_in_db) as s3conn: try: - _ = s3conn.resource.meta.client.head_object( - Bucket=s3conn.project.bucket, Key=log["path_remote"] - ) + _ = s3conn.resource.Object(s3conn.project.bucket, vals["path_remote"]) + # head_object(Bucket=s3conn.project.bucket, Key=vals["path_remote"]) except botocore.client.ClientError as err: if err.response["Error"]["Code"] == "404": errors[file] = {"error": "File not found in S3", "traceback": err.__traceback__} @@ -410,31 +416,23 @@ def update_uploaded_file_with_log(project, path_to_log_file): ) ).first() if file: - file.name_in_bucket = log["path_remote"] - file.subpath = log["subpath"] - file.project = proj_in_db.public_id - file.size_original = log["size_raw"] - file.size_stored = log["size_processed"] - file.compressed = log["compressed"] - file.public_key = log["public_key"] - file.salt = log["salt"] - file.checksum = log["checksum"] + errors[file] = {"error": "File already in database."} else: - new_file = file_schemas.NewFileSchema().load( - { - "name": file, - "name_in_bucket": log["path_remote"], - "subpath": log["subpath"], - "project": proj_in_db.public_id, - "size_original": log["size_raw"], - "size_stored": log["size_processed"], - "compressed": log["compressed"], - "public_key": log["public_key"], - "salt": log["salt"], - "checksum": log["checksum"], - } + new_file = models.File( + name=file, + name_in_bucket=vals["path_remote"], + subpath=vals["subpath"], + project_id=proj_in_db.public_id, + size_original=vals["size_raw"], + size_stored=vals["size_processed"], + compressed=vals["compressed"], + public_key=vals["public_key"], + salt=vals["salt"], + checksum=vals["checksum"], ) - files_added.append(file) + db.session.add(new_file) + files_added.append(new_file) + db.session.commit() flask.current_app.logger.info(f"Files added: {files_added}") flask.current_app.logger.info(f"Errors while adding files: {errors}") diff --git a/dds_web/api/files.py b/dds_web/api/files.py index 2e103f357..a8532976b 100644 --- a/dds_web/api/files.py +++ b/dds_web/api/files.py @@ -92,6 +92,7 @@ def post(self): ) try: + raise sqlalchemy.exc.SQLAlchemyError("test") db.session.commit() except sqlalchemy.exc.SQLAlchemyError as err: flask.current_app.logger.debug(err) From 922eb8750f7751a62cdfc4c2285951dbe1088662 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Wed, 23 Mar 2022 16:00:18 +0100 Subject: [PATCH 194/293] Update dds_web/__init__.py --- dds_web/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dds_web/__init__.py b/dds_web/__init__.py index ccf0a6fc2..7655fb8d6 100644 --- a/dds_web/__init__.py +++ b/dds_web/__init__.py @@ -383,7 +383,6 @@ def update_uploaded_file_with_log(project, path_to_log_file): """Update file details that weren't properly uploaded to db from cli log""" import botocore from dds_web.database import models - from dds_web.api.schemas.project_schemas import verify_project_exists from dds_web import db from dds_web.api.api_s3_connector import ApiS3Connector from dds_web.api.schemas import file_schemas From f45abec66a327d71dce6d3f8ab9dc419e8674612 Mon Sep 17 00:00:00 2001 From: Anandashankar Anil Date: Wed, 23 Mar 2022 16:26:58 +0100 Subject: [PATCH 195/293] Remove unused --- dds_web/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dds_web/__init__.py b/dds_web/__init__.py index 7655fb8d6..f8cc410e6 100644 --- a/dds_web/__init__.py +++ b/dds_web/__init__.py @@ -385,7 +385,6 @@ def update_uploaded_file_with_log(project, path_to_log_file): from dds_web.database import models from dds_web import db from dds_web.api.api_s3_connector import ApiS3Connector - from dds_web.api.schemas import file_schemas import json proj_in_db = models.Project.query.filter_by(public_id=project).one_or_none() From f485c6b4b6a70a191e47f342537f345998464fa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Wed, 23 Mar 2022 16:43:34 +0100 Subject: [PATCH 196/293] it works now --- dds_web/__init__.py | 17 +++++++++-------- dds_web/api/files.py | 1 - 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/dds_web/__init__.py b/dds_web/__init__.py index ccf0a6fc2..fd996d9f6 100644 --- a/dds_web/__init__.py +++ b/dds_web/__init__.py @@ -403,26 +403,27 @@ def update_uploaded_file_with_log(project, path_to_log_file): with ApiS3Connector(project=proj_in_db) as s3conn: try: - _ = s3conn.resource.Object(s3conn.project.bucket, vals["path_remote"]) - # head_object(Bucket=s3conn.project.bucket, Key=vals["path_remote"]) + _ = s3conn.resource.meta.client.head_object( + Bucket=s3conn.project.bucket, Key=vals["path_remote"] + ) except botocore.client.ClientError as err: if err.response["Error"]["Code"] == "404": errors[file] = {"error": "File not found in S3", "traceback": err.__traceback__} else: - file = models.File.query.filter( + file_object = models.File.query.filter( sqlalchemy.and_( models.File.name == sqlalchemy.func.binary(file), models.File.project_id == proj_in_db.id, ) ).first() - if file: + if file_object: errors[file] = {"error": "File already in database."} else: new_file = models.File( name=file, name_in_bucket=vals["path_remote"], subpath=vals["subpath"], - project_id=proj_in_db.public_id, + project_id=proj_in_db.id, size_original=vals["size_raw"], size_stored=vals["size_processed"], compressed=vals["compressed"], @@ -432,7 +433,7 @@ def update_uploaded_file_with_log(project, path_to_log_file): ) db.session.add(new_file) files_added.append(new_file) + db.session.commit() - db.session.commit() - flask.current_app.logger.info(f"Files added: {files_added}") - flask.current_app.logger.info(f"Errors while adding files: {errors}") + flask.current_app.logger.info(f"Files added: {files_added}") + flask.current_app.logger.info(f"Errors while adding files: {errors}") diff --git a/dds_web/api/files.py b/dds_web/api/files.py index a8532976b..2e103f357 100644 --- a/dds_web/api/files.py +++ b/dds_web/api/files.py @@ -92,7 +92,6 @@ def post(self): ) try: - raise sqlalchemy.exc.SQLAlchemyError("test") db.session.commit() except sqlalchemy.exc.SQLAlchemyError as err: flask.current_app.logger.debug(err) From 212b7a01f97a1fb3b4bfdbf3a5f87ee5c819a6ff Mon Sep 17 00:00:00 2001 From: Zishan Mirza Date: Wed, 23 Mar 2022 16:26:53 +0000 Subject: [PATCH 197/293] Validation This patch updates the validation for the project description. Signed-off-by: Zishan Mirza --- dds_web/api/schemas/project_schemas.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dds_web/api/schemas/project_schemas.py b/dds_web/api/schemas/project_schemas.py index 872c0d926..12fcd5a9c 100644 --- a/dds_web/api/schemas/project_schemas.py +++ b/dds_web/api/schemas/project_schemas.py @@ -78,9 +78,7 @@ class Meta: description = marshmallow.fields.String( required=True, allow_none=False, - validate=marshmallow.validate.And( - marshmallow.validate.Length(min=1), dds_web.utils.contains_disallowed_characters - ), + validate=marshmallow.validate.Length(min=1), error_messages={ "required": {"message": "A project description is required."}, "null": {"message": "A project description is required."}, From d2bc62c0e2f5fe9e43f1ae6f9526c342c4ae1e72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Wed, 23 Mar 2022 19:10:26 +0100 Subject: [PATCH 198/293] add validation in schema for description --- dds_web/api/project.py | 6 +++++- dds_web/api/schemas/project_schemas.py | 10 ++++++++++ dds_web/utils.py | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index b5bb729e5..d13c955cb 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -566,8 +566,12 @@ def post(self): return {"warning": warning_message} # Add a new project to db + import pymysql - new_project = project_schemas.CreateProjectSchema().load(p_info) + try: + new_project = project_schemas.CreateProjectSchema().load(p_info) + except pymysql.err.DataError as err: + raise DatabaseError(message="Unexpected database error!") db.session.add(new_project) if not new_project: diff --git a/dds_web/api/schemas/project_schemas.py b/dds_web/api/schemas/project_schemas.py index 12fcd5a9c..18032395d 100644 --- a/dds_web/api/schemas/project_schemas.py +++ b/dds_web/api/schemas/project_schemas.py @@ -6,6 +6,7 @@ # Standard Library import os +import re # Installed import botocore.client @@ -125,6 +126,15 @@ def validate_all_fields(self, data, **kwargs): ): raise marshmallow.ValidationError("Missing fields!") + @marshmallow.validates("description") + def validate_description(self, value): + """Verify that description only has words, spaces and . / ,.""" + disallowed = re.findall(r"[^(\w\s.,)]+", value) + if disallowed: + raise marshmallow.ValidationError( + message="The description can only contain letters, spaces, period and commas." + ) + def generate_bucketname(self, public_id, created_time): """Create bucket name for the given project.""" return "{pid}-{tstamp}-{rstring}".format( diff --git a/dds_web/utils.py b/dds_web/utils.py index a43bacbb1..b06b6ca5a 100644 --- a/dds_web/utils.py +++ b/dds_web/utils.py @@ -58,7 +58,7 @@ def contains_digit_or_specialchar(indata): def contains_disallowed_characters(indata): """Indatas like <9f><98><80> cause issues in Project names etc.""" - disallowed = re.findall(r"[^\w\s]+", indata) + disallowed = re.findall(r"[^(\w\s)]+", indata) if disallowed: disallowed = set(disallowed) # unique values chars = "characters" From 687508f8499f1e8f5dab242657758418c00334aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Wed, 23 Mar 2022 20:19:04 +0100 Subject: [PATCH 199/293] remove try except because not working --- dds_web/api/project.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index d13c955cb..415b17eef 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -568,10 +568,7 @@ def post(self): # Add a new project to db import pymysql - try: - new_project = project_schemas.CreateProjectSchema().load(p_info) - except pymysql.err.DataError as err: - raise DatabaseError(message="Unexpected database error!") + new_project = project_schemas.CreateProjectSchema().load(p_info) db.session.add(new_project) if not new_project: From 49f0fe9b1b9645b782e5f8d0a8c40b49140bec41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Wed, 23 Mar 2022 20:21:21 +0100 Subject: [PATCH 200/293] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba65aada8..ed27660b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,3 +58,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - Change token expiration time to 7 days (168 hours) ([#1061](https://github.com/ScilifelabDataCentre/dds_web/pull/1061)) - Add possibility of deleting invites (temporary fix in delete user endpoint) ([#1075](https://github.com/ScilifelabDataCentre/dds_web/pull/1075)) - Flask command `create-unit` to create unit without having to interact with database directly ([#1075](https://github.com/ScilifelabDataCentre/dds_web/pull/1075)) +- Let project description include . and , ([#1080](https://github.com/ScilifelabDataCentre/dds_web/pull/1080)) From 09e4021914c73199985de932f539145504dd00b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Thu, 24 Mar 2022 12:51:47 +0100 Subject: [PATCH 201/293] fix bug in flask command --- dds_web/__init__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/dds_web/__init__.py b/dds_web/__init__.py index f52f02932..33990d39f 100644 --- a/dds_web/__init__.py +++ b/dds_web/__init__.py @@ -417,6 +417,7 @@ def update_uploaded_file_with_log(project, path_to_log_file): if file_object: errors[file] = {"error": "File already in database."} else: + print(f"compressed: {vals['compressed']}") new_file = models.File( name=file, name_in_bucket=vals["path_remote"], @@ -424,11 +425,18 @@ def update_uploaded_file_with_log(project, path_to_log_file): project_id=proj_in_db.id, size_original=vals["size_raw"], size_stored=vals["size_processed"], - compressed=vals["compressed"], + compressed=not vals["compressed"], public_key=vals["public_key"], salt=vals["salt"], checksum=vals["checksum"], ) + new_version = models.Version( + size_stored=new_file.size_stored, time_uploaded=datetime.datetime.utcnow() + ) + proj_in_db.file_versions.append(new_version) + proj_in_db.files.append(new_file) + new_file.versions.append(new_version) + db.session.add(new_file) files_added.append(new_file) db.session.commit() From 20739cae10633d563213449762a4e8d6f07fe8ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Thu, 24 Mar 2022 12:53:25 +0100 Subject: [PATCH 202/293] Update dds_web/__init__.py --- dds_web/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dds_web/__init__.py b/dds_web/__init__.py index 33990d39f..066247ec4 100644 --- a/dds_web/__init__.py +++ b/dds_web/__init__.py @@ -417,7 +417,6 @@ def update_uploaded_file_with_log(project, path_to_log_file): if file_object: errors[file] = {"error": "File already in database."} else: - print(f"compressed: {vals['compressed']}") new_file = models.File( name=file, name_in_bucket=vals["path_remote"], From 02a2473b5e26fbe314332e8c1be20f07fca4fc8a Mon Sep 17 00:00:00 2001 From: Zishan Mirza Date: Thu, 24 Mar 2022 12:40:29 +0000 Subject: [PATCH 203/293] Validation This patch updates the validation for the principal investigator. Signed-off-by: Zishan Mirza --- dds_web/api/schemas/project_schemas.py | 5 +---- tests/test_project_creation.py | 6 +++++- tests/test_project_status.py | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/dds_web/api/schemas/project_schemas.py b/dds_web/api/schemas/project_schemas.py index 18032395d..5017e0029 100644 --- a/dds_web/api/schemas/project_schemas.py +++ b/dds_web/api/schemas/project_schemas.py @@ -88,10 +88,7 @@ class Meta: pi = marshmallow.fields.String( required=True, allow_none=False, - validate=marshmallow.validate.And( - marshmallow.validate.Length(min=1, max=255), - dds_web.utils.contains_disallowed_characters, - ), + validate=marshmallow.validate.Email(), error_messages={ "required": {"message": "A principal investigator is required."}, "null": {"message": "A principal investigator is required."}, diff --git a/tests/test_project_creation.py b/tests/test_project_creation.py index 8af8ff6ea..fdaa5559a 100644 --- a/tests/test_project_creation.py +++ b/tests/test_project_creation.py @@ -20,7 +20,11 @@ # CONFIG ################################################################################## CONFIG # -proj_data = {"pi": "piName", "title": "Test proj", "description": "A longer project description"} +proj_data = { + "pi": "researchuser@mailtrap.io", + "title": "Test proj", + "description": "A longer project description", +} proj_data_with_existing_users = { **proj_data, "users_to_add": [ diff --git a/tests/test_project_status.py b/tests/test_project_status.py index 6f2382487..d4e336c7c 100644 --- a/tests/test_project_status.py +++ b/tests/test_project_status.py @@ -22,7 +22,7 @@ # CONFIG ################################################################################## CONFIG # proj_data = { - "pi": "piName", + "pi": "researchuser@mailtrap.io", "title": "Test proj", "description": "A longer project description", "users_to_add": [{"email": "researchuser2@mailtrap.io", "role": "Project Owner"}], From ffa2bc8e97d30b2927e336a653c24ecc3452ad96 Mon Sep 17 00:00:00 2001 From: Valentin Georgiev <582646+valyo@users.noreply.github.com> Date: Thu, 24 Mar 2022 14:20:28 +0100 Subject: [PATCH 204/293] trying to catch sqlalchemy.exc.OperationalError at project status change --- dds_web/api/project.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 415b17eef..eadc0e059 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -124,6 +124,9 @@ def post(self): try: project.project_statuses.append(new_status_row) db.session.commit() + except (sqlalchemy.exc.OperationalError) as err: + # flask.current_app.logger.exception(err) + raise DatabaseError(message=str(err), alt_message="Database seems to be down.") except (sqlalchemy.exc.SQLAlchemyError) as err: flask.current_app.logger.exception(err) db.session.rollback() @@ -436,12 +439,17 @@ def get(self): # return ByteHours project_info.update({"Usage": proj_bhours, "Cost": proj_cost}) - project_info["Access"] = ( - models.ProjectUserKeys.query.filter_by( - project_id=p.id, user_id=current_user.username - ).count() - > 0 - ) + + try: + project_info["Access"] = ( + models.ProjectUserKeys.query.filter_by( + project_id=p.id, user_id=current_user.username + ).count() + > 0 + ) + except sqlalchemy.exc.OperationalError as err: + raise DatabaseError(message=str(err), alt_message="Database seems to be down.") + all_projects.append(project_info) return_info = { From d6eed6dc285c6221933f980bd8dc11171035d4a3 Mon Sep 17 00:00:00 2001 From: Zishan Mirza Date: Thu, 24 Mar 2022 14:13:10 +0000 Subject: [PATCH 205/293] Validation This patch adds a "try" statement. Signed-off-by: Zishan Mirza --- dds_web/api/project.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 415b17eef..27a1d9042 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -568,8 +568,11 @@ def post(self): # Add a new project to db import pymysql - new_project = project_schemas.CreateProjectSchema().load(p_info) - db.session.add(new_project) + try: + new_project = project_schemas.CreateProjectSchema().load(p_info) + db.session.add(new_project) + except sqlalchemy.exc.OperationalError as err: + raise DatabaseError(message=str(err), alt_message="Unexpected database error.") if not new_project: raise DDSArgumentError("Failed to create project.") From 652f09ba018a962994a617d2228eba5ee820f0aa Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 24 Mar 2022 15:55:36 +0100 Subject: [PATCH 206/293] Remove banner on the homepage about testing --- dds_web/templates/home.html | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/dds_web/templates/home.html b/dds_web/templates/home.html index 82264d938..4c99477b3 100644 --- a/dds_web/templates/home.html +++ b/dds_web/templates/home.html @@ -1,16 +1,6 @@ {% extends 'base.html' %} {% block body %} -{# - ######################################################### - ##### TODO: REMOVE THIS ONCE TESTING PERIOD IS OVER ##### - ######################################################### -#} -

- For information on how to use the Data Delivery System, as well as to see a testing protocol, see - here. -
-

Welcome to the SciLifeLab Data Delivery System

From 0676611856f0b2583a087220d36b69454dc6f287 Mon Sep 17 00:00:00 2001 From: Valentin Georgiev <582646+valyo@users.noreply.github.com> Date: Thu, 24 Mar 2022 17:19:22 +0100 Subject: [PATCH 207/293] try except block in the create_app --- dds_web/__init__.py | 236 ++++++++++++++++++++++---------------------- 1 file changed, 120 insertions(+), 116 deletions(-) diff --git a/dds_web/__init__.py b/dds_web/__init__.py index f52f02932..10d856c85 100644 --- a/dds_web/__init__.py +++ b/dds_web/__init__.py @@ -160,122 +160,126 @@ def setup_logging(app): def create_app(testing=False, database_uri=None): - """Construct the core application.""" - # Initiate app object - app = flask.Flask(__name__, instance_relative_config=False) - - # Default development config - app.config.from_object("dds_web.config.Config") - - # User config file, if e.g. using in production - app.config.from_envvar("DDS_APP_CONFIG", silent=True) - - # Test related configs - if database_uri is not None: - app.config["SQLALCHEMY_DATABASE_URI"] = database_uri - # Disables error catching during request handling - app.config["TESTING"] = testing - if testing: - # Simplifies testing as we don't test the session protection anyway - login_manager.session_protection = "basic" - - @app.before_request - def prepare(): - """Populate flask globals for template rendering""" - flask.g.current_user = None - if auth.current_user(): - flask.g.current_user = auth.current_user().username - elif flask_login.current_user.is_authenticated: - flask.g.current_user = flask_login.current_user.username - elif flask.request.authorization: - flask.g.current_user = flask.request.authorization.get("username") - - # Setup logging handlers - setup_logging(app) - - # Adding limiter logging - for handler in app.logger.handlers: - limiter.logger.addHandler(handler) - - # Set app.logger as the general logger - app.logger = logging.getLogger("general") - app.logger.info("Logging initiated.") - - # Initialize database - db.init_app(app) - - # Initialize mail setup - mail.init_app(app) - - # Avoid very extensive logging when sending emails - app.extensions["mail"].debug = 0 - - # Initialize marshmallows - ma.init_app(app) - - # Errors, TODO: Move somewhere else? - @app.errorhandler(sqlalchemy.exc.SQLAlchemyError) - def handle_sqlalchemyerror(e): - return f"SQLAlchemyError: {e}", 500 # TODO: Fix logging and a page - - # Initialize login manager - login_manager.init_app(app) - - @login_manager.user_loader - def load_user(user_id): - return models.User.query.get(user_id) - - if app.config["REVERSE_PROXY"]: - app.wsgi_app = ProxyFix(app.wsgi_app) - - # Initialize limiter - limiter._storage_uri = app.config.get("RATELIMIT_STORAGE_URL") - limiter.init_app(app) - - # Initialize migrations - migrate.init_app(app, db) - - # initialize OIDC - oauth.init_app(app) - oauth.register( - "default_login", - client_secret=app.config.get("OIDC_CLIENT_SECRET"), - client_id=app.config.get("OIDC_CLIENT_ID"), - server_metadata_url=app.config.get("OIDC_ACCESS_TOKEN_URL"), - client_kwargs={"scope": "openid profile email"}, - ) - - app.cli.add_command(fill_db_wrapper) - app.cli.add_command(create_new_unit) - app.cli.add_command(update_uploaded_file_with_log) - - with app.app_context(): # Everything in here has access to sessions - from dds_web.database import models - - # Need to import auth so that the modifications to the auth objects take place - import dds_web.security.auth - - # Register blueprints - from dds_web.api import api_blueprint - from dds_web.web.root import pages - from dds_web.web.user import auth_blueprint - - app.register_blueprint(api_blueprint, url_prefix="/api/v1") - app.register_blueprint(pages, url_prefix="") - app.register_blueprint(auth_blueprint, url_prefix="") - - # Set-up the schedulers - dds_web.utils.scheduler_wrapper() - - ENCRYPTION_KEY_BIT_LENGTH = 256 - ENCRYPTION_KEY_CHAR_LENGTH = int(ENCRYPTION_KEY_BIT_LENGTH / 8) - - if len(app.config.get("SECRET_KEY")) != ENCRYPTION_KEY_CHAR_LENGTH: - from dds_web.errors import KeyLengthError - - raise KeyLengthError(ENCRYPTION_KEY_CHAR_LENGTH) - - return app + try: + """Construct the core application.""" + # Initiate app object + app = flask.Flask(__name__, instance_relative_config=False) + + # Default development config + app.config.from_object("dds_web.config.Config") + + # User config file, if e.g. using in production + app.config.from_envvar("DDS_APP_CONFIG", silent=True) + + # Test related configs + if database_uri is not None: + app.config["SQLALCHEMY_DATABASE_URI"] = database_uri + # Disables error catching during request handling + app.config["TESTING"] = testing + if testing: + # Simplifies testing as we don't test the session protection anyway + login_manager.session_protection = "basic" + + @app.before_request + def prepare(): + """Populate flask globals for template rendering""" + flask.g.current_user = None + if auth.current_user(): + flask.g.current_user = auth.current_user().username + elif flask_login.current_user.is_authenticated: + flask.g.current_user = flask_login.current_user.username + elif flask.request.authorization: + flask.g.current_user = flask.request.authorization.get("username") + + # Setup logging handlers + setup_logging(app) + + # Adding limiter logging + for handler in app.logger.handlers: + limiter.logger.addHandler(handler) + + # Set app.logger as the general logger + app.logger = logging.getLogger("general") + app.logger.info("Logging initiated.") + + # Initialize database + db.init_app(app) + + # Initialize mail setup + mail.init_app(app) + + # Avoid very extensive logging when sending emails + app.extensions["mail"].debug = 0 + + # Initialize marshmallows + ma.init_app(app) + + # Errors, TODO: Move somewhere else? + @app.errorhandler(sqlalchemy.exc.SQLAlchemyError) + def handle_sqlalchemyerror(e): + return f"SQLAlchemyError: {e}", 500 # TODO: Fix logging and a page + + # Initialize login manager + login_manager.init_app(app) + + @login_manager.user_loader + def load_user(user_id): + return models.User.query.get(user_id) + + if app.config["REVERSE_PROXY"]: + app.wsgi_app = ProxyFix(app.wsgi_app) + + # Initialize limiter + limiter._storage_uri = app.config.get("RATELIMIT_STORAGE_URL") + limiter.init_app(app) + + # Initialize migrations + migrate.init_app(app, db) + + # initialize OIDC + oauth.init_app(app) + oauth.register( + "default_login", + client_secret=app.config.get("OIDC_CLIENT_SECRET"), + client_id=app.config.get("OIDC_CLIENT_ID"), + server_metadata_url=app.config.get("OIDC_ACCESS_TOKEN_URL"), + client_kwargs={"scope": "openid profile email"}, + ) + + app.cli.add_command(fill_db_wrapper) + app.cli.add_command(create_new_unit) + app.cli.add_command(update_uploaded_file_with_log) + + with app.app_context(): # Everything in here has access to sessions + from dds_web.database import models + + # Need to import auth so that the modifications to the auth objects take place + import dds_web.security.auth + + # Register blueprints + from dds_web.api import api_blueprint + from dds_web.web.root import pages + from dds_web.web.user import auth_blueprint + + app.register_blueprint(api_blueprint, url_prefix="/api/v1") + app.register_blueprint(pages, url_prefix="") + app.register_blueprint(auth_blueprint, url_prefix="") + + # Set-up the schedulers + dds_web.utils.scheduler_wrapper() + + ENCRYPTION_KEY_BIT_LENGTH = 256 + ENCRYPTION_KEY_CHAR_LENGTH = int(ENCRYPTION_KEY_BIT_LENGTH / 8) + + if len(app.config.get("SECRET_KEY")) != ENCRYPTION_KEY_CHAR_LENGTH: + from dds_web.errors import KeyLengthError + + raise KeyLengthError(ENCRYPTION_KEY_CHAR_LENGTH) + + return app + except sqlalchemy.exc.OperationalError as err: + app.logger.exception("The database seems to be down.") + sys.exit(1) @click.command("init-db") From 46b5e320c7d3bb952ccd8662ec614b8bbd962845 Mon Sep 17 00:00:00 2001 From: Valentin Georgiev <582646+valyo@users.noreply.github.com> Date: Thu, 24 Mar 2022 17:50:37 +0100 Subject: [PATCH 208/293] suggestion for catching both exceptions --- dds_web/api/project.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index eadc0e059..7adfa6534 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -124,13 +124,10 @@ def post(self): try: project.project_statuses.append(new_status_row) db.session.commit() - except (sqlalchemy.exc.OperationalError) as err: - # flask.current_app.logger.exception(err) - raise DatabaseError(message=str(err), alt_message="Database seems to be down.") - except (sqlalchemy.exc.SQLAlchemyError) as err: + except (sqlalchemy.exc.OperationalError, sqlalchemy.exc.SQLAlchemyError) as err: flask.current_app.logger.exception(err) db.session.rollback() - raise DatabaseError(message="Server Error: Status was not updated") from err + raise DatabaseError(message=str(err), alt_message="Database seems to be down." if isinstance(err, sqlalchemy.exc.OperationalError) else "Server Error: Status was not updated") # Mail users once project is made available if new_status == "Available" and send_email: From a6dedd79c87d01e32cf884488f78b5aac1d5738d Mon Sep 17 00:00:00 2001 From: Valentin Georgiev <582646+valyo@users.noreply.github.com> Date: Thu, 24 Mar 2022 18:24:19 +0100 Subject: [PATCH 209/293] import needed --- dds_web/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dds_web/__init__.py b/dds_web/__init__.py index 10d856c85..5ac3cef54 100644 --- a/dds_web/__init__.py +++ b/dds_web/__init__.py @@ -8,6 +8,7 @@ import logging import datetime import pathlib +import sys # Installed import click @@ -278,7 +279,7 @@ def load_user(user_id): return app except sqlalchemy.exc.OperationalError as err: - app.logger.exception("The database seems to be down.") + app.logger.exception("The database seems to be down. bla bla") sys.exit(1) From d7c9dc5782668fcef44601e0286dbf0a224c7f7a Mon Sep 17 00:00:00 2001 From: Valentin Georgiev <582646+valyo@users.noreply.github.com> Date: Fri, 25 Mar 2022 08:06:48 +0100 Subject: [PATCH 210/293] black and clean up --- dds_web/__init__.py | 4 ++-- dds_web/api/project.py | 22 +++++++++++++--------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/dds_web/__init__.py b/dds_web/__init__.py index 5ac3cef54..49cb3d9ec 100644 --- a/dds_web/__init__.py +++ b/dds_web/__init__.py @@ -279,8 +279,8 @@ def load_user(user_id): return app except sqlalchemy.exc.OperationalError as err: - app.logger.exception("The database seems to be down. bla bla") - sys.exit(1) + app.logger.exception("The database seems to be down.") + sys.exit(1) @click.command("init-db") diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 7adfa6534..4dfa77cd7 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -127,7 +127,12 @@ def post(self): except (sqlalchemy.exc.OperationalError, sqlalchemy.exc.SQLAlchemyError) as err: flask.current_app.logger.exception(err) db.session.rollback() - raise DatabaseError(message=str(err), alt_message="Database seems to be down." if isinstance(err, sqlalchemy.exc.OperationalError) else "Server Error: Status was not updated") + raise DatabaseError( + message=str(err), + alt_message="Database seems to be down." + if isinstance(err, sqlalchemy.exc.OperationalError) + else "Server Error: Status was not updated", + ) # Mail users once project is made available if new_status == "Available" and send_email: @@ -436,16 +441,15 @@ def get(self): # return ByteHours project_info.update({"Usage": proj_bhours, "Cost": proj_cost}) - try: - project_info["Access"] = ( - models.ProjectUserKeys.query.filter_by( - project_id=p.id, user_id=current_user.username - ).count() - > 0 - ) + project_info["Access"] = ( + models.ProjectUserKeys.query.filter_by( + project_id=p.id, user_id=current_user.username + ).count() + > 0 + ) except sqlalchemy.exc.OperationalError as err: - raise DatabaseError(message=str(err), alt_message="Database seems to be down.") + raise DatabaseError(message=str(err), alt_message="Database seems to be down.") all_projects.append(project_info) From eff7fc635c427b75cf148897b7b58b0b845b9edf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Fri, 25 Mar 2022 10:30:52 +0100 Subject: [PATCH 211/293] change 'the user does not' to 'you do not' --- dds_web/errors.py | 2 +- tests/test_user_add.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dds_web/errors.py b/dds_web/errors.py index c36c38108..f1eb66c69 100644 --- a/dds_web/errors.py +++ b/dds_web/errors.py @@ -130,7 +130,7 @@ class AccessDeniedError(LoggedHTTPException): """Errors due to incorrect project permissions.""" code = http.HTTPStatus.FORBIDDEN # 403 - description = "The user does not have the necessary permissions." + description = "You do not have the necessary permissions." def __init__( self, diff --git a/tests/test_user_add.py b/tests/test_user_add.py index fcd645719..b125544e5 100644 --- a/tests/test_user_add.py +++ b/tests/test_user_add.py @@ -831,7 +831,7 @@ def test_invite_superadmin_as_unitadmin(client): ) assert response.status_code == http.HTTPStatus.FORBIDDEN - assert "The user does not have the necessary permissions." in response.json["message"] + assert "You do not have the necessary permissions." in response.json["message"] # Invite super admin and unit admin with unit personnel @@ -846,7 +846,7 @@ def test_invite_superadmin_and_unitadmin_as_unitpersonnel(client): ) assert response.status_code == http.HTTPStatus.FORBIDDEN - assert "The user does not have the necessary permissions." in response.json["message"] + assert "You do not have the necessary permissions." in response.json["message"] # Invite super admin, unit admin or unit personnel @@ -862,7 +862,7 @@ def test_invite_superadmin_and_unitadmin_and_unitpersonnel_as_projectowner(clien ) assert response.status_code == http.HTTPStatus.FORBIDDEN - assert "The user does not have the necessary permissions." in response.json["message"] + assert "You do not have the necessary permissions." in response.json["message"] def test_invite_unituser_as_superadmin_incorrect_unit(client): From 4c7b607d0b2415b9d7df089acebca5dac166320d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Fri, 25 Mar 2022 11:33:13 +0100 Subject: [PATCH 212/293] operational error in files.py --- dds_web/api/files.py | 126 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 102 insertions(+), 24 deletions(-) diff --git a/dds_web/api/files.py b/dds_web/api/files.py index 2e103f357..6591e0683 100644 --- a/dds_web/api/files.py +++ b/dds_web/api/files.py @@ -93,10 +93,18 @@ def post(self): try: db.session.commit() - except sqlalchemy.exc.SQLAlchemyError as err: + except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err: flask.current_app.logger.debug(err) db.session.rollback() - raise DatabaseError("Failed to add new file to database.") from err + raise DatabaseError( + message=str(err), + alt_message="Failed to add new file to database" + + ( + ": Database malfunction." + if isinstance(err, sqlalchemy.exc.OperationalError) + else "." + ), + ) from err return {"message": f"File '{new_file.name}' added to db."} @@ -176,9 +184,17 @@ def put(self): db.session.add(new_version) db.session.commit() - except sqlalchemy.exc.SQLAlchemyError as err: + except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err: db.session.rollback() - raise DatabaseError(f"Failed updating file information: {err}") from err + raise DatabaseError( + message=str(err), + alt_message=f"Failed updating file information" + + ( + ": Database malfunction." + if isinstance(err, sqlalchemy.exc.OperationalError) + else "." + ), + ) from err return {"message": f"File '{file_info.get('name')}' updated in db."} @@ -205,8 +221,16 @@ def get(self): .filter(models.File.project_id == sqlalchemy.func.binary(project.id)) .all() ) - except sqlalchemy.exc.SQLAlchemyError as err: - raise DatabaseError(f"Failed to get matching files in db: {err}") from err + except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err: + raise DatabaseError( + message=str(err), + alt_message=f"Failed to get matching files in db" + + ( + ": Database malfunction." + if isinstance(err, sqlalchemy.exc.OperationalError) + else "." + ), + ) from err # The files checked are not in the db if not matching_files or matching_files is None: @@ -291,8 +315,16 @@ def get_folder_size(self, project, folder_name): .first() ) - except sqlalchemy.exc.SQLAlchemyError as err: - raise DatabaseError(message=str(err)) from err + except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err: + raise DatabaseError( + message=str(err), + alt_message=f"Could not get size of folder '{folder_name}'" + + ( + ": Database malfunction." + if isinstance(err, sqlalchemy.exc.OperationalError) + else "." + ), + ) from err return file_info.sizeSum @@ -354,8 +386,16 @@ def items_in_subpath(project, folder="."): ) distinct_folders = list(split_paths) - except sqlalchemy.exc.SQLAlchemyError as err: - raise DatabaseError(message=str(err)) from err + except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err: + raise DatabaseError( + message=str(err), + alt_message=f"Could not get items in {f'folder {folder}' if folder != '.' else 'root'}" + + ( + ": Database malfunction." + if isinstance(err, sqlalchemy.exc.OperationalError) + else "." + ), + ) from err else: return distinct_files, distinct_folders @@ -404,9 +444,25 @@ def delete_multiple(self, project, files): db.session.rollback() not_exist_list.append(entry) continue - except (sqlalchemy.exc.SQLAlchemyError, DatabaseError) as err: + except ( + sqlalchemy.exc.SQLAlchemyError, + DatabaseError, + sqlalchemy.exc.OperationalError, + ) as err: db.session.rollback() - not_removed_dict[entry] = str(err) + flask.current_app.logger.exception(err) + not_removed_dict[entry] = ( + str(err) + if isinstance(err, DatabaseError) + else ( + "Could not collect the remote item name" + + ( + ": Database malfunction." + if isinstance(err, sqlalchemy.exc.OperationalError) + else "." + ) + ) + ) continue # Remove from s3 bucket @@ -420,9 +476,14 @@ def delete_multiple(self, project, files): # Commit to db if ok try: db.session.commit() - except sqlalchemy.exc.SQLAlchemyError as err: + except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err: db.session.rollback() - not_removed_dict[entry] = str(err) + flask.current_app.logger.exception(err) + not_removed_dict[entry] = "Could not remove data" + ( + ": Database malfunction." + if isinstance(err, sqlalchemy.exc.OperationalError) + else "." + ) continue return not_removed_dict, not_exist_list @@ -505,13 +566,14 @@ def delete(self): except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err: db.session.rollback() flask.current_app.logger.error( - "Files deleted in S3 but not in db. The entries must be synchronised!" + "Files deleted in S3 but not in db. The entries must be synchronised! " + f"Error: {str(err)}" + ) + not_removed[folder_name] = "Could not remove files in folder" + ( + ": Database malfunction." + if isinstance(err, sqlalchemy.exc.OperationalError) + else "." ) - if isinstance(err, sqlalchemy.exc.OperationalError): - err_msg = "Database malfunction." - else: - err_msg = str(err) - not_removed[folder_name] = err_msg fail_type = "db" break @@ -541,8 +603,16 @@ def get_files_for_deletion(self, project: str, folder: str): ) .all() ) - except sqlalchemy.exc.SQLAlchemyError as err: - raise DatabaseError(message=str(err)) from err + except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err: + raise DatabaseError( + message=str(err), + alt_message="Could not collect files for deletion" + + ( + ": Database malfunction." + if isinstance(err, sqlalchemy.exc.OperationalError) + else "." + ), + ) from err return files @@ -651,10 +721,18 @@ def put(self): raise NoSuchFileError() file.time_latest_download = dds_web.utils.current_time() - except sqlalchemy.exc.SQLAlchemyError as err: + except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err: db.session.rollback() flask.current_app.logger.exception(str(err)) - raise DatabaseError("Update of file info failed.") from err + raise DatabaseError( + message=str(err), + alt_message="Update of file info failed" + + ( + ": Database malfunction." + if isinstance(err, sqlalchemy.exc.OperationalError) + else "." + ), + ) from err else: # flask.current_app.logger.debug("File %s updated", file_name) db.session.commit() From acaadfeb3005c6a05d08055aae38a4e5f6939183 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Fri, 25 Mar 2022 11:48:10 +0100 Subject: [PATCH 213/293] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed27660b0..419278937 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,3 +59,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - Add possibility of deleting invites (temporary fix in delete user endpoint) ([#1075](https://github.com/ScilifelabDataCentre/dds_web/pull/1075)) - Flask command `create-unit` to create unit without having to interact with database directly ([#1075](https://github.com/ScilifelabDataCentre/dds_web/pull/1075)) - Let project description include . and , ([#1080](https://github.com/ScilifelabDataCentre/dds_web/pull/1080)) +- Catch OperationalError if there is a database malfunction in `files.py` ([#1089](https://github.com/ScilifelabDataCentre/dds_web/pull/1089)) From 808aa6143482e32a97350fd68fbb6e383354ab14 Mon Sep 17 00:00:00 2001 From: Valentin Georgiev <582646+valyo@users.noreply.github.com> Date: Fri, 25 Mar 2022 12:41:14 +0100 Subject: [PATCH 214/293] operational error in class CreateProject --- dds_web/api/project.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 4dfa77cd7..be82ed6d1 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -597,10 +597,15 @@ def post(self): try: db.session.commit() - except (sqlalchemy.exc.SQLAlchemyError, TypeError) as err: + except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError, TypeError) as err: flask.current_app.logger.exception(err) db.session.rollback() - raise DatabaseError(message="Server Error: Project was not created") from err + raise DatabaseError( + message=str(err), + alt_message="Server Error: Project was not created" + (": Database malfunction.") + if isinstance(err, sqlalchemy.exc.OperationalError) + else ".", + ) from err except ( marshmallow.exceptions.ValidationError, DDSArgumentError, From 8bf74a4ef06f2309c586b090505eea6da8496641 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 25 Mar 2022 12:48:29 +0100 Subject: [PATCH 215/293] Add docs link to top navbar --- dds_web/templates/navbar.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dds_web/templates/navbar.html b/dds_web/templates/navbar.html index 94dbf17e6..7d3dd35ae 100644 --- a/dds_web/templates/navbar.html +++ b/dds_web/templates/navbar.html @@ -14,9 +14,9 @@
  • - + - About + Documentation
    • {% if g.current_user %} From fc57df169fed56670df21ace342bb188be39fb82 Mon Sep 17 00:00:00 2001 From: Valentin Georgiev <582646+valyo@users.noreply.github.com> Date: Fri, 25 Mar 2022 12:51:28 +0100 Subject: [PATCH 216/293] small fixes --- dds_web/api/project.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index be82ed6d1..1850624b0 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -132,7 +132,7 @@ def post(self): alt_message="Database seems to be down." if isinstance(err, sqlalchemy.exc.OperationalError) else "Server Error: Status was not updated", - ) + ) from err # Mail users once project is made available if new_status == "Available" and send_email: @@ -449,7 +449,12 @@ def get(self): > 0 ) except sqlalchemy.exc.OperationalError as err: - raise DatabaseError(message=str(err), alt_message="Database seems to be down.") + raise DatabaseError( + message=str(err), + alt_message="Database seems to be down." + if isinstance(err, sqlalchemy.exc.OperationalError) + else ".", + ) from err all_projects.append(project_info) From e09c8a6e30e8245e9cd43b263e60fa9fdfb3f9a9 Mon Sep 17 00:00:00 2001 From: valyo <582646+valyo@users.noreply.github.com> Date: Fri, 25 Mar 2022 13:07:27 +0100 Subject: [PATCH 217/293] Update dds_web/api/project.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ina Odén Österbo <35953392+inaod568@users.noreply.github.com> --- dds_web/api/project.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 1850624b0..d7933ef7e 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -129,10 +129,15 @@ def post(self): db.session.rollback() raise DatabaseError( message=str(err), - alt_message="Database seems to be down." - if isinstance(err, sqlalchemy.exc.OperationalError) - else "Server Error: Status was not updated", - ) from err + alt_message=( + "Status not updated" + + ( + "Database malfunction." + if isinstance(err, sqlalchemy.exc.OperationalError) + else "Server Error: Status was not updated" + ) + ) + ) from err # Mail users once project is made available if new_status == "Available" and send_email: From 5cee5ee8994de723f82a87438f62342d5ad53efe Mon Sep 17 00:00:00 2001 From: valyo <582646+valyo@users.noreply.github.com> Date: Fri, 25 Mar 2022 13:14:43 +0100 Subject: [PATCH 218/293] Update dds_web/api/project.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ina Odén Österbo <35953392+inaod568@users.noreply.github.com> --- dds_web/api/project.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index d7933ef7e..9a8585388 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -612,9 +612,12 @@ def post(self): db.session.rollback() raise DatabaseError( message=str(err), - alt_message="Server Error: Project was not created" + (": Database malfunction.") - if isinstance(err, sqlalchemy.exc.OperationalError) - else ".", + alt_message=( + "Project was not created" + + (": Database malfunction." + if isinstance(err, sqlalchemy.exc.OperationalError) + else ": Server error."), + ) ) from err except ( marshmallow.exceptions.ValidationError, From be1f9984cae28da4fbd8d7dacc54ba81e595aeaa Mon Sep 17 00:00:00 2001 From: valyo <582646+valyo@users.noreply.github.com> Date: Fri, 25 Mar 2022 13:15:13 +0100 Subject: [PATCH 219/293] Update dds_web/api/project.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ina Odén Österbo <35953392+inaod568@users.noreply.github.com> --- dds_web/api/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 9a8585388..43a954179 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -453,7 +453,7 @@ def get(self): ).count() > 0 ) - except sqlalchemy.exc.OperationalError as err: + except (sqlalchemy.exc.OperationalError, sqlalchemy.exc.SQLAlchemyError) as err: raise DatabaseError( message=str(err), alt_message="Database seems to be down." From 30e543f4e716227f8b2a1b40472445285ac134bd Mon Sep 17 00:00:00 2001 From: Valentin Georgiev <582646+valyo@users.noreply.github.com> Date: Fri, 25 Mar 2022 13:16:27 +0100 Subject: [PATCH 220/293] alt_message fix --- dds_web/api/project.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 43a954179..022edd02e 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -130,14 +130,14 @@ def post(self): raise DatabaseError( message=str(err), alt_message=( - "Status not updated" + - ( - "Database malfunction." + "Status was not updated" + + ( + "Database malfunction." if isinstance(err, sqlalchemy.exc.OperationalError) else "Server Error: Status was not updated" - ) - ) - ) from err + ) + ), + ) from err # Mail users once project is made available if new_status == "Available" and send_email: From 666dae8fe6ded06e9584282f1ec1a635917163d6 Mon Sep 17 00:00:00 2001 From: Valentin Georgiev <582646+valyo@users.noreply.github.com> Date: Fri, 25 Mar 2022 13:22:26 +0100 Subject: [PATCH 221/293] black --- dds_web/api/project.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 022edd02e..c1701654a 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -456,9 +456,14 @@ def get(self): except (sqlalchemy.exc.OperationalError, sqlalchemy.exc.SQLAlchemyError) as err: raise DatabaseError( message=str(err), - alt_message="Database seems to be down." - if isinstance(err, sqlalchemy.exc.OperationalError) - else ".", + alt_message=( + "Could not get users project access information." + + ( + ": Database malfunction." + if isinstance(err, sqlalchemy.exc.OperationalError) + else "." + ), + ), ) from err all_projects.append(project_info) @@ -613,11 +618,13 @@ def post(self): raise DatabaseError( message=str(err), alt_message=( - "Project was not created" - + (": Database malfunction." + "Project was not created" + + ( + ": Database malfunction." if isinstance(err, sqlalchemy.exc.OperationalError) - else ": Server error."), - ) + else ": Server error." + ), + ), ) from err except ( marshmallow.exceptions.ValidationError, From c426e84d1dd5f0d5052da615971653297476f2cb Mon Sep 17 00:00:00 2001 From: Zishan Mirza Date: Fri, 25 Mar 2022 12:31:35 +0000 Subject: [PATCH 222/293] Validation This patch updates CHANGELOG.md. Signed-off-by: Zishan Mirza --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed27660b0..7b37e1a35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,3 +59,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - Add possibility of deleting invites (temporary fix in delete user endpoint) ([#1075](https://github.com/ScilifelabDataCentre/dds_web/pull/1075)) - Flask command `create-unit` to create unit without having to interact with database directly ([#1075](https://github.com/ScilifelabDataCentre/dds_web/pull/1075)) - Let project description include . and , ([#1080](https://github.com/ScilifelabDataCentre/dds_web/pull/1080)) +- Switched the validation for the principal investigator from string to email ([#1084](https://github.com/ScilifelabDataCentre/dds_web/pull/1084)). From 5c7e3737b6ade0de362dc73805f3dade6b17ff1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Fri, 25 Mar 2022 13:36:12 +0100 Subject: [PATCH 223/293] Update dds_web/api/files.py Co-authored-by: Anandashankar Anil --- dds_web/api/files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/api/files.py b/dds_web/api/files.py index 6591e0683..19641e08b 100644 --- a/dds_web/api/files.py +++ b/dds_web/api/files.py @@ -455,7 +455,7 @@ def delete_multiple(self, project, files): str(err) if isinstance(err, DatabaseError) else ( - "Could not collect the remote item name" + "Could not collect the remote file name" + ( ": Database malfunction." if isinstance(err, sqlalchemy.exc.OperationalError) From daf317cdf2ca725cae54afe2d3446ebd82a989cc Mon Sep 17 00:00:00 2001 From: Valentin Georgiev <582646+valyo@users.noreply.github.com> Date: Fri, 25 Mar 2022 14:05:17 +0100 Subject: [PATCH 224/293] catch OperationalError in delete_project_contents --- dds_web/api/project.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index c1701654a..9be3a1b4a 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -558,13 +558,22 @@ def delete_project_contents(project): models.Version.time_deleted.is_(None), ) ).update({"time_deleted": dds_web.utils.current_time()}) - except (sqlalchemy.exc.SQLAlchemyError, AttributeError) as sqlerr: + except ( + sqlalchemy.exc.SQLAlchemyError, + sqlalchemy.exc.OperationalError, + AttributeError, + ) as sqlerr: raise DeletionError( project=project.public_id, message=str(sqlerr), alt_message=( "Project bucket contents were deleted, but they were not deleted from the " "database. Please contact SciLifeLab Data Centre." + + ( + "Database malfunction." + if isinstance(err, sqlalchemy.exc.OperationalError) + else "." + ) ), ) from sqlerr From 3e63adba58daee3a9bb75c47750246709d793ef0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Fri, 25 Mar 2022 14:20:22 +0100 Subject: [PATCH 225/293] catch operationalerror in s3 --- dds_web/api/s3.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/dds_web/api/s3.py b/dds_web/api/s3.py index 0c5c7486c..e34100270 100644 --- a/dds_web/api/s3.py +++ b/dds_web/api/s3.py @@ -42,8 +42,16 @@ def get(self): try: sfsp_proj, keys, url, bucketname = ApiS3Connector(project=project).get_s3_info() - except sqlalchemy.exc.SQLAlchemyError as sqlerr: - raise DatabaseError(message=str(sqlerr)) from sqlerr + except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as sqlerr: + raise DatabaseError( + message=str(sqlerr), + alt_message="Could not get cloud information" + + ( + ": Database malfunction." + if isinstance(sqlerr, sqlalchemy.exc.OperationalError) + else "." + ), + ) from sqlerr if any(x is None for x in [url, keys, bucketname]): raise S3ProjectNotFoundError("No s3 info returned!") From 764ec191793e53a3fb7f3babbdaeadb3b62e3514 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Fri, 25 Mar 2022 14:33:19 +0100 Subject: [PATCH 226/293] only need to add operationalerror to handle_db_error --- dds_web/api/dds_decorators.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dds_web/api/dds_decorators.py b/dds_web/api/dds_decorators.py index 57e5925af..509ceda6e 100644 --- a/dds_web/api/dds_decorators.py +++ b/dds_web/api/dds_decorators.py @@ -110,8 +110,11 @@ def perform_get(*args, **kwargs): # Run function, catch errors try: result = func(*args, **kwargs) - except sqlalchemy.exc.SQLAlchemyError as sqlerr: - raise DatabaseError(message=str(sqlerr)) from sqlerr + except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as sqlerr: + flask.current_app.logger.exception(sqlerr) + raise DatabaseError( + message=str(sqlerr), alt_message="Database malfunction!" + ) from sqlerr return result From 658c995aa4404b4ef2a9ffb69ff49f1ddb84fe92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Fri, 25 Mar 2022 14:35:17 +0100 Subject: [PATCH 227/293] Update dds_web/api/dds_decorators.py --- dds_web/api/dds_decorators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/api/dds_decorators.py b/dds_web/api/dds_decorators.py index 509ceda6e..8074f10e9 100644 --- a/dds_web/api/dds_decorators.py +++ b/dds_web/api/dds_decorators.py @@ -113,7 +113,7 @@ def perform_get(*args, **kwargs): except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as sqlerr: flask.current_app.logger.exception(sqlerr) raise DatabaseError( - message=str(sqlerr), alt_message="Database malfunction!" + message=str(sqlerr), alt_message="Database malfunction!" if isinstance(sqlerr, sqlalchemy.exc.OperationalError) else None ) from sqlerr return result From 9a0d55f41d0ca926740be1ff2925c021daf977d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Fri, 25 Mar 2022 14:35:42 +0100 Subject: [PATCH 228/293] linting --- dds_web/api/dds_decorators.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dds_web/api/dds_decorators.py b/dds_web/api/dds_decorators.py index 8074f10e9..68185df3b 100644 --- a/dds_web/api/dds_decorators.py +++ b/dds_web/api/dds_decorators.py @@ -113,7 +113,10 @@ def perform_get(*args, **kwargs): except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as sqlerr: flask.current_app.logger.exception(sqlerr) raise DatabaseError( - message=str(sqlerr), alt_message="Database malfunction!" if isinstance(sqlerr, sqlalchemy.exc.OperationalError) else None + message=str(sqlerr), + alt_message="Database malfunction!" + if isinstance(sqlerr, sqlalchemy.exc.OperationalError) + else None, ) from sqlerr return result From a04d1a7d04ec0a458007b64051bdc124a5536cfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Fri, 25 Mar 2022 14:36:19 +0100 Subject: [PATCH 229/293] () --- dds_web/api/dds_decorators.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dds_web/api/dds_decorators.py b/dds_web/api/dds_decorators.py index 68185df3b..b01395ba0 100644 --- a/dds_web/api/dds_decorators.py +++ b/dds_web/api/dds_decorators.py @@ -114,9 +114,11 @@ def perform_get(*args, **kwargs): flask.current_app.logger.exception(sqlerr) raise DatabaseError( message=str(sqlerr), - alt_message="Database malfunction!" - if isinstance(sqlerr, sqlalchemy.exc.OperationalError) - else None, + alt_message=( + "Database malfunction!" + if isinstance(sqlerr, sqlalchemy.exc.OperationalError) + else None + ), ) from sqlerr return result From 02668ede53d6740fe764d92d6b0eb976007b1160 Mon Sep 17 00:00:00 2001 From: valyo <582646+valyo@users.noreply.github.com> Date: Fri, 25 Mar 2022 14:58:18 +0100 Subject: [PATCH 230/293] Update dds_web/api/project.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ina Odén Österbo <35953392+inaod568@users.noreply.github.com> --- dds_web/api/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 9be3a1b4a..307badbf7 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -134,7 +134,7 @@ def post(self): + ( "Database malfunction." if isinstance(err, sqlalchemy.exc.OperationalError) - else "Server Error: Status was not updated" + else ": Server Error." ) ), ) from err From ba8cffc9cb020319bdfc6b745006bfa6640b8739 Mon Sep 17 00:00:00 2001 From: valyo <582646+valyo@users.noreply.github.com> Date: Fri, 25 Mar 2022 14:58:25 +0100 Subject: [PATCH 231/293] Update dds_web/api/project.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ina Odén Österbo <35953392+inaod568@users.noreply.github.com> --- dds_web/api/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 307badbf7..03e40a5e6 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -132,7 +132,7 @@ def post(self): alt_message=( "Status was not updated" + ( - "Database malfunction." + ": Database malfunction." if isinstance(err, sqlalchemy.exc.OperationalError) else ": Server Error." ) From 9cf65fa82441d4097918722facc470f7074db03a Mon Sep 17 00:00:00 2001 From: Zishan Mirza Date: Fri, 25 Mar 2022 14:03:43 +0000 Subject: [PATCH 232/293] Validation Switched the principal investigator from string to email. Signed-off-by: Zishan Mirza --- dds_web/development/db_init.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dds_web/development/db_init.py b/dds_web/development/db_init.py index cfa70c732..8e55c45d2 100644 --- a/dds_web/development/db_init.py +++ b/dds_web/development/db_init.py @@ -84,7 +84,7 @@ def fill_db(): title="First Project", description="This is a test project. You will be able to upload to but NOT download " "from this project. Create a new project to test the entire system. ", - pi="PI Name", + pi="support@example.com", bucket="testbucket", ) @@ -104,7 +104,7 @@ def fill_db(): title="Second Project", description="This is a test project. You will be able to upload to but NOT download " "from this project. Create a new project to test the entire system. ", - pi="PI Name", + pi="support@example.com", bucket=f"secondproject-{str(dds_web.utils.timestamp(ts_format='%Y%m%d%H%M%S'))}-{str(uuid.uuid4())}", ) From 282661b349784728014b235895e02bfb3c62104e Mon Sep 17 00:00:00 2001 From: Valentin Georgiev <582646+valyo@users.noreply.github.com> Date: Thu, 24 Mar 2022 14:20:28 +0100 Subject: [PATCH 233/293] trying to catch sqlalchemy.exc.OperationalError at project status change --- dds_web/api/project.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 415b17eef..eadc0e059 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -124,6 +124,9 @@ def post(self): try: project.project_statuses.append(new_status_row) db.session.commit() + except (sqlalchemy.exc.OperationalError) as err: + # flask.current_app.logger.exception(err) + raise DatabaseError(message=str(err), alt_message="Database seems to be down.") except (sqlalchemy.exc.SQLAlchemyError) as err: flask.current_app.logger.exception(err) db.session.rollback() @@ -436,12 +439,17 @@ def get(self): # return ByteHours project_info.update({"Usage": proj_bhours, "Cost": proj_cost}) - project_info["Access"] = ( - models.ProjectUserKeys.query.filter_by( - project_id=p.id, user_id=current_user.username - ).count() - > 0 - ) + + try: + project_info["Access"] = ( + models.ProjectUserKeys.query.filter_by( + project_id=p.id, user_id=current_user.username + ).count() + > 0 + ) + except sqlalchemy.exc.OperationalError as err: + raise DatabaseError(message=str(err), alt_message="Database seems to be down.") + all_projects.append(project_info) return_info = { From 9a1eff078a720094764dde2cbfe6ca8f8eff99f7 Mon Sep 17 00:00:00 2001 From: Valentin Georgiev <582646+valyo@users.noreply.github.com> Date: Thu, 24 Mar 2022 17:19:22 +0100 Subject: [PATCH 234/293] try except block in the create_app --- dds_web/__init__.py | 236 ++++++++++++++++++++++---------------------- 1 file changed, 120 insertions(+), 116 deletions(-) diff --git a/dds_web/__init__.py b/dds_web/__init__.py index 066247ec4..dd3780f25 100644 --- a/dds_web/__init__.py +++ b/dds_web/__init__.py @@ -160,122 +160,126 @@ def setup_logging(app): def create_app(testing=False, database_uri=None): - """Construct the core application.""" - # Initiate app object - app = flask.Flask(__name__, instance_relative_config=False) - - # Default development config - app.config.from_object("dds_web.config.Config") - - # User config file, if e.g. using in production - app.config.from_envvar("DDS_APP_CONFIG", silent=True) - - # Test related configs - if database_uri is not None: - app.config["SQLALCHEMY_DATABASE_URI"] = database_uri - # Disables error catching during request handling - app.config["TESTING"] = testing - if testing: - # Simplifies testing as we don't test the session protection anyway - login_manager.session_protection = "basic" - - @app.before_request - def prepare(): - """Populate flask globals for template rendering""" - flask.g.current_user = None - if auth.current_user(): - flask.g.current_user = auth.current_user().username - elif flask_login.current_user.is_authenticated: - flask.g.current_user = flask_login.current_user.username - elif flask.request.authorization: - flask.g.current_user = flask.request.authorization.get("username") - - # Setup logging handlers - setup_logging(app) - - # Adding limiter logging - for handler in app.logger.handlers: - limiter.logger.addHandler(handler) - - # Set app.logger as the general logger - app.logger = logging.getLogger("general") - app.logger.info("Logging initiated.") - - # Initialize database - db.init_app(app) - - # Initialize mail setup - mail.init_app(app) - - # Avoid very extensive logging when sending emails - app.extensions["mail"].debug = 0 - - # Initialize marshmallows - ma.init_app(app) - - # Errors, TODO: Move somewhere else? - @app.errorhandler(sqlalchemy.exc.SQLAlchemyError) - def handle_sqlalchemyerror(e): - return f"SQLAlchemyError: {e}", 500 # TODO: Fix logging and a page - - # Initialize login manager - login_manager.init_app(app) - - @login_manager.user_loader - def load_user(user_id): - return models.User.query.get(user_id) - - if app.config["REVERSE_PROXY"]: - app.wsgi_app = ProxyFix(app.wsgi_app) - - # Initialize limiter - limiter._storage_uri = app.config.get("RATELIMIT_STORAGE_URL") - limiter.init_app(app) - - # Initialize migrations - migrate.init_app(app, db) - - # initialize OIDC - oauth.init_app(app) - oauth.register( - "default_login", - client_secret=app.config.get("OIDC_CLIENT_SECRET"), - client_id=app.config.get("OIDC_CLIENT_ID"), - server_metadata_url=app.config.get("OIDC_ACCESS_TOKEN_URL"), - client_kwargs={"scope": "openid profile email"}, - ) - - app.cli.add_command(fill_db_wrapper) - app.cli.add_command(create_new_unit) - app.cli.add_command(update_uploaded_file_with_log) - - with app.app_context(): # Everything in here has access to sessions - from dds_web.database import models - - # Need to import auth so that the modifications to the auth objects take place - import dds_web.security.auth - - # Register blueprints - from dds_web.api import api_blueprint - from dds_web.web.root import pages - from dds_web.web.user import auth_blueprint - - app.register_blueprint(api_blueprint, url_prefix="/api/v1") - app.register_blueprint(pages, url_prefix="") - app.register_blueprint(auth_blueprint, url_prefix="") - - # Set-up the schedulers - dds_web.utils.scheduler_wrapper() - - ENCRYPTION_KEY_BIT_LENGTH = 256 - ENCRYPTION_KEY_CHAR_LENGTH = int(ENCRYPTION_KEY_BIT_LENGTH / 8) - - if len(app.config.get("SECRET_KEY")) != ENCRYPTION_KEY_CHAR_LENGTH: - from dds_web.errors import KeyLengthError - - raise KeyLengthError(ENCRYPTION_KEY_CHAR_LENGTH) - - return app + try: + """Construct the core application.""" + # Initiate app object + app = flask.Flask(__name__, instance_relative_config=False) + + # Default development config + app.config.from_object("dds_web.config.Config") + + # User config file, if e.g. using in production + app.config.from_envvar("DDS_APP_CONFIG", silent=True) + + # Test related configs + if database_uri is not None: + app.config["SQLALCHEMY_DATABASE_URI"] = database_uri + # Disables error catching during request handling + app.config["TESTING"] = testing + if testing: + # Simplifies testing as we don't test the session protection anyway + login_manager.session_protection = "basic" + + @app.before_request + def prepare(): + """Populate flask globals for template rendering""" + flask.g.current_user = None + if auth.current_user(): + flask.g.current_user = auth.current_user().username + elif flask_login.current_user.is_authenticated: + flask.g.current_user = flask_login.current_user.username + elif flask.request.authorization: + flask.g.current_user = flask.request.authorization.get("username") + + # Setup logging handlers + setup_logging(app) + + # Adding limiter logging + for handler in app.logger.handlers: + limiter.logger.addHandler(handler) + + # Set app.logger as the general logger + app.logger = logging.getLogger("general") + app.logger.info("Logging initiated.") + + # Initialize database + db.init_app(app) + + # Initialize mail setup + mail.init_app(app) + + # Avoid very extensive logging when sending emails + app.extensions["mail"].debug = 0 + + # Initialize marshmallows + ma.init_app(app) + + # Errors, TODO: Move somewhere else? + @app.errorhandler(sqlalchemy.exc.SQLAlchemyError) + def handle_sqlalchemyerror(e): + return f"SQLAlchemyError: {e}", 500 # TODO: Fix logging and a page + + # Initialize login manager + login_manager.init_app(app) + + @login_manager.user_loader + def load_user(user_id): + return models.User.query.get(user_id) + + if app.config["REVERSE_PROXY"]: + app.wsgi_app = ProxyFix(app.wsgi_app) + + # Initialize limiter + limiter._storage_uri = app.config.get("RATELIMIT_STORAGE_URL") + limiter.init_app(app) + + # Initialize migrations + migrate.init_app(app, db) + + # initialize OIDC + oauth.init_app(app) + oauth.register( + "default_login", + client_secret=app.config.get("OIDC_CLIENT_SECRET"), + client_id=app.config.get("OIDC_CLIENT_ID"), + server_metadata_url=app.config.get("OIDC_ACCESS_TOKEN_URL"), + client_kwargs={"scope": "openid profile email"}, + ) + + app.cli.add_command(fill_db_wrapper) + app.cli.add_command(create_new_unit) + app.cli.add_command(update_uploaded_file_with_log) + + with app.app_context(): # Everything in here has access to sessions + from dds_web.database import models + + # Need to import auth so that the modifications to the auth objects take place + import dds_web.security.auth + + # Register blueprints + from dds_web.api import api_blueprint + from dds_web.web.root import pages + from dds_web.web.user import auth_blueprint + + app.register_blueprint(api_blueprint, url_prefix="/api/v1") + app.register_blueprint(pages, url_prefix="") + app.register_blueprint(auth_blueprint, url_prefix="") + + # Set-up the schedulers + dds_web.utils.scheduler_wrapper() + + ENCRYPTION_KEY_BIT_LENGTH = 256 + ENCRYPTION_KEY_CHAR_LENGTH = int(ENCRYPTION_KEY_BIT_LENGTH / 8) + + if len(app.config.get("SECRET_KEY")) != ENCRYPTION_KEY_CHAR_LENGTH: + from dds_web.errors import KeyLengthError + + raise KeyLengthError(ENCRYPTION_KEY_CHAR_LENGTH) + + return app + except sqlalchemy.exc.OperationalError as err: + app.logger.exception("The database seems to be down.") + sys.exit(1) @click.command("init-db") From 53610c082b4a7838277ce4aa14df4d5dd58bb03a Mon Sep 17 00:00:00 2001 From: Valentin Georgiev <582646+valyo@users.noreply.github.com> Date: Thu, 24 Mar 2022 17:50:37 +0100 Subject: [PATCH 235/293] suggestion for catching both exceptions --- dds_web/api/project.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index eadc0e059..7adfa6534 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -124,13 +124,10 @@ def post(self): try: project.project_statuses.append(new_status_row) db.session.commit() - except (sqlalchemy.exc.OperationalError) as err: - # flask.current_app.logger.exception(err) - raise DatabaseError(message=str(err), alt_message="Database seems to be down.") - except (sqlalchemy.exc.SQLAlchemyError) as err: + except (sqlalchemy.exc.OperationalError, sqlalchemy.exc.SQLAlchemyError) as err: flask.current_app.logger.exception(err) db.session.rollback() - raise DatabaseError(message="Server Error: Status was not updated") from err + raise DatabaseError(message=str(err), alt_message="Database seems to be down." if isinstance(err, sqlalchemy.exc.OperationalError) else "Server Error: Status was not updated") # Mail users once project is made available if new_status == "Available" and send_email: From ec6238d90c5e230881c3e410c59685373150b405 Mon Sep 17 00:00:00 2001 From: Valentin Georgiev <582646+valyo@users.noreply.github.com> Date: Thu, 24 Mar 2022 18:24:19 +0100 Subject: [PATCH 236/293] import needed --- dds_web/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dds_web/__init__.py b/dds_web/__init__.py index dd3780f25..48bfc92a6 100644 --- a/dds_web/__init__.py +++ b/dds_web/__init__.py @@ -8,6 +8,7 @@ import logging import datetime import pathlib +import sys # Installed import click @@ -278,7 +279,7 @@ def load_user(user_id): return app except sqlalchemy.exc.OperationalError as err: - app.logger.exception("The database seems to be down.") + app.logger.exception("The database seems to be down. bla bla") sys.exit(1) From 068cbecabd7ce602c0d71b7dd1b3dd352933f286 Mon Sep 17 00:00:00 2001 From: Valentin Georgiev <582646+valyo@users.noreply.github.com> Date: Fri, 25 Mar 2022 08:06:48 +0100 Subject: [PATCH 237/293] black and clean up --- dds_web/__init__.py | 4 ++-- dds_web/api/project.py | 22 +++++++++++++--------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/dds_web/__init__.py b/dds_web/__init__.py index 48bfc92a6..0a80c8f64 100644 --- a/dds_web/__init__.py +++ b/dds_web/__init__.py @@ -279,8 +279,8 @@ def load_user(user_id): return app except sqlalchemy.exc.OperationalError as err: - app.logger.exception("The database seems to be down. bla bla") - sys.exit(1) + app.logger.exception("The database seems to be down.") + sys.exit(1) @click.command("init-db") diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 7adfa6534..4dfa77cd7 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -127,7 +127,12 @@ def post(self): except (sqlalchemy.exc.OperationalError, sqlalchemy.exc.SQLAlchemyError) as err: flask.current_app.logger.exception(err) db.session.rollback() - raise DatabaseError(message=str(err), alt_message="Database seems to be down." if isinstance(err, sqlalchemy.exc.OperationalError) else "Server Error: Status was not updated") + raise DatabaseError( + message=str(err), + alt_message="Database seems to be down." + if isinstance(err, sqlalchemy.exc.OperationalError) + else "Server Error: Status was not updated", + ) # Mail users once project is made available if new_status == "Available" and send_email: @@ -436,16 +441,15 @@ def get(self): # return ByteHours project_info.update({"Usage": proj_bhours, "Cost": proj_cost}) - try: - project_info["Access"] = ( - models.ProjectUserKeys.query.filter_by( - project_id=p.id, user_id=current_user.username - ).count() - > 0 - ) + project_info["Access"] = ( + models.ProjectUserKeys.query.filter_by( + project_id=p.id, user_id=current_user.username + ).count() + > 0 + ) except sqlalchemy.exc.OperationalError as err: - raise DatabaseError(message=str(err), alt_message="Database seems to be down.") + raise DatabaseError(message=str(err), alt_message="Database seems to be down.") all_projects.append(project_info) From c5a91723a37faea70904d533840501f7e5c71b77 Mon Sep 17 00:00:00 2001 From: Valentin Georgiev <582646+valyo@users.noreply.github.com> Date: Fri, 25 Mar 2022 12:41:14 +0100 Subject: [PATCH 238/293] operational error in class CreateProject --- dds_web/api/project.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 4dfa77cd7..be82ed6d1 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -597,10 +597,15 @@ def post(self): try: db.session.commit() - except (sqlalchemy.exc.SQLAlchemyError, TypeError) as err: + except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError, TypeError) as err: flask.current_app.logger.exception(err) db.session.rollback() - raise DatabaseError(message="Server Error: Project was not created") from err + raise DatabaseError( + message=str(err), + alt_message="Server Error: Project was not created" + (": Database malfunction.") + if isinstance(err, sqlalchemy.exc.OperationalError) + else ".", + ) from err except ( marshmallow.exceptions.ValidationError, DDSArgumentError, From 669396d9b4b05cb8261a611697b7e21456ea8afe Mon Sep 17 00:00:00 2001 From: Valentin Georgiev <582646+valyo@users.noreply.github.com> Date: Fri, 25 Mar 2022 12:51:28 +0100 Subject: [PATCH 239/293] small fixes --- dds_web/api/project.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index be82ed6d1..1850624b0 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -132,7 +132,7 @@ def post(self): alt_message="Database seems to be down." if isinstance(err, sqlalchemy.exc.OperationalError) else "Server Error: Status was not updated", - ) + ) from err # Mail users once project is made available if new_status == "Available" and send_email: @@ -449,7 +449,12 @@ def get(self): > 0 ) except sqlalchemy.exc.OperationalError as err: - raise DatabaseError(message=str(err), alt_message="Database seems to be down.") + raise DatabaseError( + message=str(err), + alt_message="Database seems to be down." + if isinstance(err, sqlalchemy.exc.OperationalError) + else ".", + ) from err all_projects.append(project_info) From 5172a660b9266131f094b911b303b239f97e5e32 Mon Sep 17 00:00:00 2001 From: valyo <582646+valyo@users.noreply.github.com> Date: Fri, 25 Mar 2022 13:07:27 +0100 Subject: [PATCH 240/293] Update dds_web/api/project.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ina Odén Österbo <35953392+inaod568@users.noreply.github.com> --- dds_web/api/project.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 1850624b0..d7933ef7e 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -129,10 +129,15 @@ def post(self): db.session.rollback() raise DatabaseError( message=str(err), - alt_message="Database seems to be down." - if isinstance(err, sqlalchemy.exc.OperationalError) - else "Server Error: Status was not updated", - ) from err + alt_message=( + "Status not updated" + + ( + "Database malfunction." + if isinstance(err, sqlalchemy.exc.OperationalError) + else "Server Error: Status was not updated" + ) + ) + ) from err # Mail users once project is made available if new_status == "Available" and send_email: From cd840c3209435d262b585ceb0338da626fda049b Mon Sep 17 00:00:00 2001 From: valyo <582646+valyo@users.noreply.github.com> Date: Fri, 25 Mar 2022 13:14:43 +0100 Subject: [PATCH 241/293] Update dds_web/api/project.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ina Odén Österbo <35953392+inaod568@users.noreply.github.com> --- dds_web/api/project.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index d7933ef7e..9a8585388 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -612,9 +612,12 @@ def post(self): db.session.rollback() raise DatabaseError( message=str(err), - alt_message="Server Error: Project was not created" + (": Database malfunction.") - if isinstance(err, sqlalchemy.exc.OperationalError) - else ".", + alt_message=( + "Project was not created" + + (": Database malfunction." + if isinstance(err, sqlalchemy.exc.OperationalError) + else ": Server error."), + ) ) from err except ( marshmallow.exceptions.ValidationError, From e65e3c2dfef271efe1257be01a2343d497d6ebb4 Mon Sep 17 00:00:00 2001 From: valyo <582646+valyo@users.noreply.github.com> Date: Fri, 25 Mar 2022 13:15:13 +0100 Subject: [PATCH 242/293] Update dds_web/api/project.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ina Odén Österbo <35953392+inaod568@users.noreply.github.com> --- dds_web/api/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 9a8585388..43a954179 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -453,7 +453,7 @@ def get(self): ).count() > 0 ) - except sqlalchemy.exc.OperationalError as err: + except (sqlalchemy.exc.OperationalError, sqlalchemy.exc.SQLAlchemyError) as err: raise DatabaseError( message=str(err), alt_message="Database seems to be down." From 03728b8d320e3a4e7f753381403e773d6ac24aec Mon Sep 17 00:00:00 2001 From: Valentin Georgiev <582646+valyo@users.noreply.github.com> Date: Fri, 25 Mar 2022 13:16:27 +0100 Subject: [PATCH 243/293] alt_message fix --- dds_web/api/project.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 43a954179..022edd02e 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -130,14 +130,14 @@ def post(self): raise DatabaseError( message=str(err), alt_message=( - "Status not updated" + - ( - "Database malfunction." + "Status was not updated" + + ( + "Database malfunction." if isinstance(err, sqlalchemy.exc.OperationalError) else "Server Error: Status was not updated" - ) - ) - ) from err + ) + ), + ) from err # Mail users once project is made available if new_status == "Available" and send_email: From d8e148072e01614eb1fb11e86a491f427c0e9d01 Mon Sep 17 00:00:00 2001 From: Valentin Georgiev <582646+valyo@users.noreply.github.com> Date: Fri, 25 Mar 2022 13:22:26 +0100 Subject: [PATCH 244/293] black --- dds_web/api/project.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 022edd02e..c1701654a 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -456,9 +456,14 @@ def get(self): except (sqlalchemy.exc.OperationalError, sqlalchemy.exc.SQLAlchemyError) as err: raise DatabaseError( message=str(err), - alt_message="Database seems to be down." - if isinstance(err, sqlalchemy.exc.OperationalError) - else ".", + alt_message=( + "Could not get users project access information." + + ( + ": Database malfunction." + if isinstance(err, sqlalchemy.exc.OperationalError) + else "." + ), + ), ) from err all_projects.append(project_info) @@ -613,11 +618,13 @@ def post(self): raise DatabaseError( message=str(err), alt_message=( - "Project was not created" - + (": Database malfunction." + "Project was not created" + + ( + ": Database malfunction." if isinstance(err, sqlalchemy.exc.OperationalError) - else ": Server error."), - ) + else ": Server error." + ), + ), ) from err except ( marshmallow.exceptions.ValidationError, From 7af5dd83743b64e77af6286fe891fc36505b6488 Mon Sep 17 00:00:00 2001 From: Valentin Georgiev <582646+valyo@users.noreply.github.com> Date: Fri, 25 Mar 2022 14:05:17 +0100 Subject: [PATCH 245/293] catch OperationalError in delete_project_contents --- dds_web/api/project.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index c1701654a..9be3a1b4a 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -558,13 +558,22 @@ def delete_project_contents(project): models.Version.time_deleted.is_(None), ) ).update({"time_deleted": dds_web.utils.current_time()}) - except (sqlalchemy.exc.SQLAlchemyError, AttributeError) as sqlerr: + except ( + sqlalchemy.exc.SQLAlchemyError, + sqlalchemy.exc.OperationalError, + AttributeError, + ) as sqlerr: raise DeletionError( project=project.public_id, message=str(sqlerr), alt_message=( "Project bucket contents were deleted, but they were not deleted from the " "database. Please contact SciLifeLab Data Centre." + + ( + "Database malfunction." + if isinstance(err, sqlalchemy.exc.OperationalError) + else "." + ) ), ) from sqlerr From d80f1604b2b6df94f88f1577b742d9377f215ee7 Mon Sep 17 00:00:00 2001 From: valyo <582646+valyo@users.noreply.github.com> Date: Fri, 25 Mar 2022 14:58:18 +0100 Subject: [PATCH 246/293] Update dds_web/api/project.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ina Odén Österbo <35953392+inaod568@users.noreply.github.com> --- dds_web/api/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 9be3a1b4a..307badbf7 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -134,7 +134,7 @@ def post(self): + ( "Database malfunction." if isinstance(err, sqlalchemy.exc.OperationalError) - else "Server Error: Status was not updated" + else ": Server Error." ) ), ) from err From fd9bd11a6623316391b173804d127c22cc3af118 Mon Sep 17 00:00:00 2001 From: valyo <582646+valyo@users.noreply.github.com> Date: Fri, 25 Mar 2022 14:58:25 +0100 Subject: [PATCH 247/293] Update dds_web/api/project.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ina Odén Österbo <35953392+inaod568@users.noreply.github.com> --- dds_web/api/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 307badbf7..03e40a5e6 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -132,7 +132,7 @@ def post(self): alt_message=( "Status was not updated" + ( - "Database malfunction." + ": Database malfunction." if isinstance(err, sqlalchemy.exc.OperationalError) else ": Server Error." ) From 848a0c2a137a3f7df6db8aa6dfe27b6f3a574653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Fri, 25 Mar 2022 15:48:47 +0100 Subject: [PATCH 248/293] add operational error --- dds_web/api/db_tools.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/dds_web/api/db_tools.py b/dds_web/api/db_tools.py index de5627767..74113fa00 100644 --- a/dds_web/api/db_tools.py +++ b/dds_web/api/db_tools.py @@ -34,8 +34,18 @@ def remove_user_self_deletion_request(user): email = request_row.email db.session.delete(request_row) db.session.commit() - except sqlalchemy.exc.SQLAlchemyError as err: + except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err: db.session.rollback() - raise DatabaseError(message=str(err)) from err + raise DatabaseError( + message=str(err), + alt_message=( + "Failed to remove deletion request" + + ( + ": Database malfunction." + if isinstance(err, sqlalchemy.exc.OperationalError) + else "." + ) + ), + ) from err return email From be8445ecf90240f5c994e4882998b398bf11a4b8 Mon Sep 17 00:00:00 2001 From: Anandashankar Anil Date: Fri, 25 Mar 2022 16:50:52 +0100 Subject: [PATCH 249/293] Catch sqlalchemy.exc.OperationalError --- dds_web/api/user.py | 108 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 84 insertions(+), 24 deletions(-) diff --git a/dds_web/api/user.py b/dds_web/api/user.py index 74d7b4443..648550f90 100644 --- a/dds_web/api/user.py +++ b/dds_web/api/user.py @@ -262,9 +262,17 @@ def invite_user(email, new_user_role, project=None, unit=None): if goahead: try: db.session.commit() - except sqlalchemy.exc.SQLAlchemyError as sqlerr: + except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as sqlerr: db.session.rollback() - raise ddserr.DatabaseError(message=str(sqlerr)) + raise ddserr.DatabaseError( + message=str(sqlerr), + alt_message=f"Invitation failed" + + ( + ": Database malfunction." + if isinstance(sqlerr, sqlalchemy.exc.OperationalError) + else "." + ), + ) from sqlerr AddUser.compose_and_send_email_to_user( userobj=new_invite, mail_type="invite", link=link @@ -374,12 +382,22 @@ def add_to_project(whom, project, role, send_email=True): try: db.session.commit() - except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.IntegrityError) as err: + except ( + sqlalchemy.exc.SQLAlchemyError, + sqlalchemy.exc.IntegrityError, + sqlalchemy.exc.OperationalError, + ) as err: flask.current_app.logger.exception(err) db.session.rollback() raise ddserr.DatabaseError( - message="Server Error: User was not associated with the project" - ) + message=str(err), + alt_message=f"Server Error: User was not associated with the project" + + ( + ": Database malfunction." + if isinstance(err, sqlalchemy.exc.OperationalError) + else "." + ), + ) from err # If project is already released and not expired, send mail to user send_email = send_email and project.current_status == "Available" @@ -549,11 +567,17 @@ def delete(self): "status": http.HTTPStatus.OK, } - except sqlalchemy.exc.SQLAlchemyError as sqlerr: + except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as sqlerr: db.session.rollback() raise ddserr.DatabaseError( - message=f"Creation of self-deletion request failed due to database error: {sqlerr}", - ) + message=str(sqlerr), + alt_message=f"Creation of self-deletion request failed" + + ( + ": Database malfunction." + if isinstance(sqlerr, sqlalchemy.exc.OperationalError) + else "." + ), + ) from sqlerr # Create link for deletion request email link = flask.url_for("auth_blueprint.confirm_self_deletion", token=token, _external=True) @@ -688,9 +712,17 @@ def post(self): try: db.session.commit() - except sqlalchemy.exc.SQLAlchemyError as err: + except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err: db.session.rollback() - raise ddserr.DatabaseError(message=str(err)) + raise ddserr.DatabaseError( + message=str(err), + alt_message=f"Unexpected database error" + + ( + ": Database malfunction." + if isinstance(err, sqlalchemy.exc.OperationalError) + else "." + ), + ) from err msg = ( f"The user account {user.username} ({user_email_str}, {user.role}) " f" has been {action}d successfully been by {current_user.name} ({current_user.role})." @@ -793,9 +825,17 @@ def delete_user(user): parent_user = models.User.query.get(user.username) db.session.delete(parent_user) db.session.commit() - except sqlalchemy.exc.SQLAlchemyError as err: + except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err: db.session.rollback() - raise ddserr.DatabaseError(message=str(err)) + raise ddserr.DatabaseError( + message=str(err), + alt_message=f"Failed to delete user" + + ( + ": Database malfunction." + if isinstance(err, sqlalchemy.exc.OperationalError) + else "." + ), + ) from err @staticmethod def delete_invite(email): @@ -819,11 +859,15 @@ def delete_invite(email): "The invite connected to the email " f"{email or '[no email provided]'} was not deleted." ) - if isinstance(err, sqlalchemy.exc.OperationalError): - err_msg = "Database malfunction." - else: - err_msg = str(err) - raise ddserr.DatabaseError(message=err_msg) from err + raise ddserr.DatabaseError( + message=str(err), + alt_message=f"Failed to delete invite" + + ( + ": Database malfunction." + if isinstance(err, sqlalchemy.exc.OperationalError) + else "." + ), + ) from err return email @@ -872,14 +916,22 @@ def post(self): try: db.session.commit() - except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.IntegrityError) as err: + except ( + sqlalchemy.exc.SQLAlchemyError, + sqlalchemy.exc.IntegrityError, + sqlalchemy.exc.OperationalError, + ) as err: flask.current_app.logger.exception(err) db.session.rollback() raise ddserr.DatabaseError( - message=( - "Server Error: Removing user association with the project has not succeeded" - ) - ) + message=str(err), + alt_message=f"Server Error: Removing user association with the project has not succeeded" + + ( + ": Database malfunction." + if isinstance(err, sqlalchemy.exc.OperationalError) + else "." + ), + ) from err flask.current_app.logger.debug( f"User {existing_user.username} no longer associated with project {project.public_id}." @@ -946,9 +998,17 @@ def get(self): unit_info = models.Unit.query.filter( models.Unit.id == sqlalchemy.func.binary(current_user.unit_id) ).first() - except sqlalchemy.exc.SQLAlchemyError as err: + except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as err: flask.current_app.logger.exception(err) - raise ddserr.DatabaseError("Failed getting unit information.") + raise ddserr.DatabaseError( + message=str(err), + alt_message=f"Failed to get unit information." + + ( + ": Database malfunction." + if isinstance(err, sqlalchemy.exc.OperationalError) + else "." + ), + ) from err # Total number of GB hours and cost saved in the db for the specific unit total_gbhours_db = 0.0 From ff3a02471bc4d71ae19a14fc755da01fcd97418e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= Date: Fri, 25 Mar 2022 17:48:04 +0100 Subject: [PATCH 250/293] last I think --- dds_web/api/dds_decorators.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/dds_web/api/dds_decorators.py b/dds_web/api/dds_decorators.py index b01395ba0..57ba2bb0a 100644 --- a/dds_web/api/dds_decorators.py +++ b/dds_web/api/dds_decorators.py @@ -145,8 +145,18 @@ def init_resource(self, *args, **kwargs): aws_access_key_id=self.keys["access_key"], aws_secret_access_key=self.keys["secret_key"], ) - except sqlalchemy.exc.SQLAlchemyError as sqlerr: - raise DatabaseError(message=str(sqlerr)) from sqlerr + except (sqlalchemy.exc.SQLAlchemyError, sqlalchemy.exc.OperationalError) as sqlerr: + raise DatabaseError( + message=str(sqlerr), + alt_message=( + "Could not connect to cloud" + + ( + ": Database malfunction." + if isinstance(sqlerr, sqlalchemy.exc.OperationalError) + else "." + ), + ), + ) from sqlerr except botocore.client.ClientError as clierr: raise S3ConnectionError(message=str(clierr)) from clierr From 12951f5b7938b2a2df330dff4412253577641dfb Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 25 Mar 2022 21:04:26 +0100 Subject: [PATCH 251/293] Better front-end styling for 2fa confirm page --- dds_web/forms.py | 2 +- dds_web/templates/user/confirm2fa.html | 50 +++++++++++++++----------- 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/dds_web/forms.py b/dds_web/forms.py index 41f81e6b6..4958387c1 100644 --- a/dds_web/forms.py +++ b/dds_web/forms.py @@ -84,7 +84,7 @@ class LogoutForm(flask_wtf.FlaskForm): class Confirm2FACodeForm(flask_wtf.FlaskForm): hotp = wtforms.StringField( - "hotp", + "Multi-factor authentication code", validators=[wtforms.validators.InputRequired(), wtforms.validators.Length(min=8, max=8)], ) submit = wtforms.SubmitField("Authenticate") diff --git a/dds_web/templates/user/confirm2fa.html b/dds_web/templates/user/confirm2fa.html index 759d33461..33a880009 100644 --- a/dds_web/templates/user/confirm2fa.html +++ b/dds_web/templates/user/confirm2fa.html @@ -6,32 +6,42 @@ {% block body %} -

    Please enter your one-time authentication code.

    -

    Please complete the login by entering the one-time authentication code that was sent to you. - The one-time codes are valid for a short time (15 minutes) after they have been issued. -

    -
    - {{ form.csrf_token }} +

    Please enter your one-time authentication code.

    +

    Please complete the login by entering the one-time authentication code that was sent to you. + The one-time codes are valid for a short time (15 minutes) after they have been issued. +

    + + {{ form.csrf_token }} + +
    - {{ form.hotp.label }} - {{ form.hotp }} -
      + {{ form.hotp.label(class="col-md-auto mb-2 col-form-label") }} +
      + {{ form.hotp(class="form-control mb-2") }} +
      + + +
      + +
      + + {% if form.hotp.errors %} {% for error in form.hotp.errors %} -
    • - {{ error }} -
      • +
      {{ error }}
      {% endfor %} -
    + {% endif %} - - {{ form.submit }} +
    -
    + -
    - {{ cancel_form.csrf_token }} - {{ cancel_form.cancel }} -
    +
    + {{ cancel_form.csrf_token }} + {{ cancel_form.cancel(class="btn btn-link ps-0") }} +
    {% endblock %} From 6850548b768294d78de0edfb916742d2e628041a Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 25 Mar 2022 21:06:11 +0100 Subject: [PATCH 252/293] Remove mb-2 from label --- dds_web/templates/user/confirm2fa.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/templates/user/confirm2fa.html b/dds_web/templates/user/confirm2fa.html index 33a880009..314ebeda1 100644 --- a/dds_web/templates/user/confirm2fa.html +++ b/dds_web/templates/user/confirm2fa.html @@ -16,7 +16,7 @@
    - {{ form.hotp.label(class="col-md-auto mb-2 col-form-label") }} + {{ form.hotp.label(class="col-md-auto col-form-label") }}
    {{ form.hotp(class="form-control mb-2") }}
    From 3ac510d378267687f6beeb371d803e0a998b0dad Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 25 Mar 2022 20:43:49 +0000 Subject: [PATCH 253/293] First test of adding a gitpod config --- .gitpod.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .gitpod.yml diff --git a/.gitpod.yml b/.gitpod.yml new file mode 100644 index 000000000..e9200eef7 --- /dev/null +++ b/.gitpod.yml @@ -0,0 +1,33 @@ +--- +# based on https://github.com/gitpod-io/template-docker-compose + +tasks: + - init: docker-compose pull + - command: chmod a+x /workspace/dds_web && docker-compose up + +ports: + - port: 5000 + onOpen: open-preview + - port: 1080 + onOpen: open-preview + +vscode: + extensions: + - ms-azuretools.vscode-docker + - esbenp.prettier-vscode +github: + prebuilds: + # enable for the default branch (defaults to true) + master: true + # enable for all branches in this repo (defaults to false) + branches: false + # enable for pull requests coming from this repo (defaults to true) + pullRequests: true + # enable for pull requests coming from forks (defaults to false) + pullRequestsFromForks: true + # add a check to pull requests (defaults to true) + addCheck: true + # add a "Review in Gitpod" button as a comment to pull requests (defaults to false) + addComment: false + # add a "Review in Gitpod" button to the pull request's description (defaults to false) + addBadge: true \ No newline at end of file From 8093d3377fd4da4da05f5b0ca6dc7d7a950a4c7a Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 25 Mar 2022 20:57:16 +0000 Subject: [PATCH 254/293] Test and fix gitpod config --- .gitpod.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index e9200eef7..7a6e72f16 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -2,8 +2,8 @@ # based on https://github.com/gitpod-io/template-docker-compose tasks: - - init: docker-compose pull - - command: chmod a+x /workspace/dds_web && docker-compose up + - init: docker compose build --pull && git clone https://github.com/ScilifelabDataCentre/dds_cli.git ../dds_cli + - command: chmod a+x /workspace/dds_web && docker compose --profile cli up ports: - port: 5000 From ea9a524e4bd9f7a0d85c57ff5bb8ba64ae29048b Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sat, 26 Mar 2022 15:05:36 +0000 Subject: [PATCH 255/293] fix linting --- .gitpod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index 7a6e72f16..d57a5183c 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -30,4 +30,4 @@ github: # add a "Review in Gitpod" button as a comment to pull requests (defaults to false) addComment: false # add a "Review in Gitpod" button to the pull request's description (defaults to false) - addBadge: true \ No newline at end of file + addBadge: true From ed673848a1201d0a2d1a89e1e0f90dc8398d0a82 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 28 Mar 2022 07:36:11 +0000 Subject: [PATCH 256/293] Improvements to gitpod config - Used additionalRepositories to fetch cli code instead of just pulling - Keep init and command in same terminal - Define all open ports and ignore the ones we don't care about - Open site in a tab instead of preview --- .gitpod.yml | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index d57a5183c..22a14204e 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,20 +1,35 @@ --- # based on https://github.com/gitpod-io/template-docker-compose +# multi-repo +additionalRepositories: + - url: https://github.com/ScilifelabDataCentre/dds_cli + checkoutLocation: dds_cli + tasks: - - init: docker compose build --pull && git clone https://github.com/ScilifelabDataCentre/dds_cli.git ../dds_cli - - command: chmod a+x /workspace/dds_web && docker compose --profile cli up + - name: Build backend and run server + init: > + chmod a+x /workspace/ && + docker compose build --pull + command: docker compose --profile cli up ports: - - port: 5000 - onOpen: open-preview - - port: 1080 - onOpen: open-preview + - port: 5000 # backend + onOpen: open-browser + - port: 1080 # mailcatcher + onOpen: open-browser + - port: 9000 # minio + onOpen: ignore + - port: 9001 # minio + onOpen: ignore + - port: 3306 # db + onOpen: ignore vscode: extensions: - ms-azuretools.vscode-docker - esbenp.prettier-vscode + github: prebuilds: # enable for the default branch (defaults to true) From 25b1bcb5cdf705d60381473980cafabc990f05c4 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 28 Mar 2022 07:50:20 +0000 Subject: [PATCH 257/293] Add vscode settings.json - Fix prettier in GitPod YAML - Add aVSCode settings file The settings file means GitPod VSCode opens without a welcome page and automatically formats with prettier on save. --- .gitpod.yml | 12 ++++++------ .vscode/settings.json | 5 +++++ 2 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.gitpod.yml b/.gitpod.yml index 22a14204e..cc3098d5f 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -7,22 +7,22 @@ additionalRepositories: checkoutLocation: dds_cli tasks: - - name: Build backend and run server + - name: Build backend and run server init: > chmod a+x /workspace/ && docker compose build --pull command: docker compose --profile cli up ports: - - port: 5000 # backend + - port: 5000 # backend onOpen: open-browser - - port: 1080 # mailcatcher + - port: 1080 # mailcatcher onOpen: open-browser - - port: 9000 # minio + - port: 9000 # minio onOpen: ignore - - port: 9001 # minio + - port: 9001 # minio onOpen: ignore - - port: 3306 # db + - port: 3306 # db onOpen: ignore vscode: diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..eeee976e3 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.formatOnSave": true, + "workbench.startupEditor": "none" +} From 2de42299402a9edcced9320bdc3f0b0ad11f1891 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 28 Mar 2022 11:03:18 +0000 Subject: [PATCH 258/293] The new browser tab is quite annoying --- .gitpod.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index cc3098d5f..43872bff5 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -15,9 +15,9 @@ tasks: ports: - port: 5000 # backend - onOpen: open-browser + onOpen: open-preview - port: 1080 # mailcatcher - onOpen: open-browser + onOpen: open-preview - port: 9000 # minio onOpen: ignore - port: 9001 # minio From e19ed78e798e7065acaddea18167947a1e47a4d0 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 28 Mar 2022 11:04:01 +0000 Subject: [PATCH 259/293] backend also depends on node_builder --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index ece3acba6..dc707e306 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -59,6 +59,8 @@ services: depends_on: db: condition: service_healthy + node_builder: + condition: service_healthy restart: on-failure ports: - 127.0.0.1:5000:5000 From b6aba410a207494cf9b941ff3e87f4f0dacf1468 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 28 Mar 2022 11:31:11 +0000 Subject: [PATCH 260/293] Remove dependency on node_builder --- docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index dc707e306..ece3acba6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -59,8 +59,6 @@ services: depends_on: db: condition: service_healthy - node_builder: - condition: service_healthy restart: on-failure ports: - 127.0.0.1:5000:5000 From 06f138aec9fff338c5ec514578d956e2ced381b2 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 28 Mar 2022 11:49:22 +0000 Subject: [PATCH 261/293] Some tweaks to hopefully get port forwarding working more smoothly --- .gitpod.yml | 5 +++++ .vscode/settings.json | 9 ++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index 43872bff5..4525562ac 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -16,14 +16,19 @@ tasks: ports: - port: 5000 # backend onOpen: open-preview + visibility: public - port: 1080 # mailcatcher onOpen: open-preview + visibility: public - port: 9000 # minio onOpen: ignore + visibility: public - port: 9001 # minio onOpen: ignore + visibility: public - port: 3306 # db onOpen: ignore + visibility: public vscode: extensions: diff --git a/.vscode/settings.json b/.vscode/settings.json index eeee976e3..bb653234b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,12 @@ { "editor.defaultFormatter": "esbenp.prettier-vscode", "editor.formatOnSave": true, - "workbench.startupEditor": "none" + "workbench.startupEditor": "none", + "remote.SSH.defaultForwardedPorts": [ + { + "localPort": 5000, + "name": "ddsweb", + "remotePort": 5000 + } + ] } From f7bed04d378b13176a465d8871993b999bdbc33b Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 28 Mar 2022 12:40:46 +0000 Subject: [PATCH 262/293] Styled several forms - password reset etc. --- dds_web/templates/user/change_password.html | 76 ++++++++++--------- dds_web/templates/user/confirm2fa.html | 3 +- dds_web/templates/user/logout.html | 7 +- .../user/password_reset_completed.html | 32 ++++---- .../user/request_reset_password.html | 33 ++++---- dds_web/templates/user/reset_password.html | 48 +++++++----- 6 files changed, 112 insertions(+), 87 deletions(-) diff --git a/dds_web/templates/user/change_password.html b/dds_web/templates/user/change_password.html index cb5d3c298..d09ab263e 100644 --- a/dds_web/templates/user/change_password.html +++ b/dds_web/templates/user/change_password.html @@ -6,45 +6,53 @@ {% block body %} -
    - {{ form.csrf_token }} - - {{ form.current_password.label }} - {{ form.current_password }} -
      + + {{ form.csrf_token }} + + +
      + {{ form.current_password.label(class="col-md-2 col-form-label") }} +
      + {{ form.current_password(class="form-control mb-2") }} +
      + {% if form.current_password.errors %} {% for error in form.current_password.errors %} -
    • - {{ error }} -
      • +
      {{ error }}
      {% endfor %} -
    - - - {{ form.new_password.label }} - {{ form.new_password }} -
      + {% endif %} +
    + + +
    + {{ form.new_password.label(class="col-md-2 col-form-label") }} +
    + {{ form.new_password(class="form-control mb-2") }} +
    + {% if form.new_password.errors %} {% for error in form.new_password.errors %} -
  • - {{ error }} -
    • +
    {{ error }}
    {% endfor %} - - - - {{ form.confirm_new_password.label }} - {{ form.confirm_new_password }} -
      + {% endif %} +
    + + +
    + {{ form.confirm_new_password.label(class="col-md-2 col-form-label") }} +
    + {{ form.confirm_new_password(class="form-control mb-2") }} +
    + {% if form.confirm_new_password.errors %} {% for error in form.confirm_new_password.errors %} -
  • - {{ error }} -
    • +
    {{ error }}
    {% endfor %} - - - - - {{ form.submit }} - - + {% endif %} +
    + + + + {% endblock %} diff --git a/dds_web/templates/user/confirm2fa.html b/dds_web/templates/user/confirm2fa.html index 314ebeda1..de34c7df0 100644 --- a/dds_web/templates/user/confirm2fa.html +++ b/dds_web/templates/user/confirm2fa.html @@ -8,8 +8,7 @@

    Please enter your one-time authentication code.

    Please complete the login by entering the one-time authentication code that was sent to you. - The one-time codes are valid for a short time (15 minutes) after they have been issued. -

    + The one-time codes are valid for a short time (15 minutes) after they have been issued.

    {{ form.csrf_token }} diff --git a/dds_web/templates/user/logout.html b/dds_web/templates/user/logout.html index b2da403af..a5a146255 100644 --- a/dds_web/templates/user/logout.html +++ b/dds_web/templates/user/logout.html @@ -5,9 +5,12 @@ {%- endblock %} {% block body %} + +

    Click the button below to log out of the DDS web interface.

    + {{ form.csrf_token }} - {{ form.logout }} + {{ form.logout(class="btn btn-success") }}
    -Change Password + {% endblock %} diff --git a/dds_web/templates/user/password_reset_completed.html b/dds_web/templates/user/password_reset_completed.html index 403546ecd..b99e3f2c4 100644 --- a/dds_web/templates/user/password_reset_completed.html +++ b/dds_web/templates/user/password_reset_completed.html @@ -7,29 +7,29 @@ {% block body %} {% if units_to_contact %} -

    - You have lost access to your active projects when you reset your - password. -

    -

    - Please contact the units where you have active projects to restore your - access. -

    +
    +
    Lost access
    +

    + Warning: + You have lost access to your active projects when you reset your password. +

    +

    + Please contact the units where you have active projects to restore your access. +

    +
    -

    You have active projects in

    -

    +

    You have active projects in:

      {% for unit_name, contact in units_to_contact.items()%}
    • {{ unit_name }}: {{ contact }}
      • {% endfor %}
    {% endif %} - - + +

    + Back to Login - +

    - {% endblock %} +{% endblock %} diff --git a/dds_web/templates/user/request_reset_password.html b/dds_web/templates/user/request_reset_password.html index 5d8db4d02..b3cab1874 100644 --- a/dds_web/templates/user/request_reset_password.html +++ b/dds_web/templates/user/request_reset_password.html @@ -6,24 +6,31 @@ {% block body %} -
    - {{ form.csrf_token }} + + {{ form.csrf_token }} +
    + - {{ form.email.label }} - {{ form.email }} -
      + {{ form.email.label(class="col-md-auto col-form-label") }} +
      + {{ form.email(class="form-control mb-2", type="email") }} +
      + {% if form.email.errors %} {% for error in form.email.errors %} -
    • - {{ error }} -
      • +
      {{ error }}
      {% endfor %} -
    - + {% endif %} - {{ form.submit }} - - +
    + +
    +
    + + {% endblock %} diff --git a/dds_web/templates/user/reset_password.html b/dds_web/templates/user/reset_password.html index 8a8c6c82f..dfdd5ba73 100644 --- a/dds_web/templates/user/reset_password.html +++ b/dds_web/templates/user/reset_password.html @@ -6,35 +6,43 @@ {% block body %} -
    - {{ form.csrf_token }} + + {{ form.csrf_token }} + +
    - {{ form.password.label }} - {{ form.password }} -
      + {{ form.password.label(class="col-md-2 col-form-label") }} +
      + {{ form.password(class="form-control mb-2") }} +
      + {% if form.password.errors %} {% for error in form.password.errors %} -
    • - {{ error }} -
      • +
      {{ error }}
      {% endfor %} -
    + {% endif %} + +
    +
    - {{ form.confirm_password.label }} - {{ form.confirm_password }} -
      + {{ form.confirm_password.label(class="col-md-2 col-form-label") }} +
      + {{ form.confirm_password(class="form-control mb-2") }} +
      + {% if form.confirm_password.errors %} {% for error in form.confirm_password.errors %} -
    • - {{ error }} -
      • +
      {{ error }}
      {% endfor %} -
    - + {% endif %} +
    - - {{ form.submit }} + + -
    + {% endblock %} From eef12c364a4e4d6c19fad441bbcfc0468504706b Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 28 Mar 2022 13:38:56 +0000 Subject: [PATCH 263/293] Remove the vscode settings port forwarding --- .gitpod.yml | 2 +- .vscode/settings.json | 9 +-------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index 4525562ac..079a809df 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -15,7 +15,7 @@ tasks: ports: - port: 5000 # backend - onOpen: open-preview + onOpen: open-browser visibility: public - port: 1080 # mailcatcher onOpen: open-preview diff --git a/.vscode/settings.json b/.vscode/settings.json index bb653234b..eeee976e3 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,12 +1,5 @@ { "editor.defaultFormatter": "esbenp.prettier-vscode", "editor.formatOnSave": true, - "workbench.startupEditor": "none", - "remote.SSH.defaultForwardedPorts": [ - { - "localPort": 5000, - "name": "ddsweb", - "remotePort": 5000 - } - ] + "workbench.startupEditor": "none" } From d48ea7a9114f20a19867d66a18f7969f05dd3414 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 28 Mar 2022 14:48:55 +0000 Subject: [PATCH 264/293] Scaffold new projects page --- dds_web/templates/home.html | 2 +- dds_web/templates/navbar.html | 8 ++------ dds_web/templates/user/projects.html | 10 ++++++++++ dds_web/web/user.py | 9 +++++++++ 4 files changed, 22 insertions(+), 7 deletions(-) create mode 100644 dds_web/templates/user/projects.html diff --git a/dds_web/templates/home.html b/dds_web/templates/home.html index 4c99477b3..af73e7ed7 100644 --- a/dds_web/templates/home.html +++ b/dds_web/templates/home.html @@ -8,7 +8,7 @@

    Welcome to the SciLifeLab Data Delivery System

    {% if g.current_user %} - + View projects diff --git a/dds_web/templates/navbar.html b/dds_web/templates/navbar.html index 7d3dd35ae..2d0d55a45 100644 --- a/dds_web/templates/navbar.html +++ b/dds_web/templates/navbar.html @@ -21,13 +21,9 @@ {% if g.current_user %}
  • - + - {% if g.is_admin %} - Dashboard - {% else %} - Projects - {% endif %} + Projects
  • diff --git a/dds_web/templates/user/projects.html b/dds_web/templates/user/projects.html new file mode 100644 index 000000000..9f057bd97 --- /dev/null +++ b/dds_web/templates/user/projects.html @@ -0,0 +1,10 @@ +{% extends "base.html" %} + +{% block page_title -%} +Your projects +{%- endblock %} + +{% block body %} + + +{% endblock %} \ No newline at end of file diff --git a/dds_web/web/user.py b/dds_web/web/user.py index 7cc97dd32..d33cdcabc 100644 --- a/dds_web/web/user.py +++ b/dds_web/web/user.py @@ -560,3 +560,12 @@ def account_info(): """User account page""" return flask.render_template("user/account.html", account_info={}, enumerate=enumerate) + + +@auth_blueprint.route("/projects", methods=["GET"]) +@flask_login.login_required +@logging_bind_request +def projects_info(): + """User projects page""" + + return flask.render_template("user/projects.html", enumerate=enumerate) \ No newline at end of file From 5e557b75dbc30fabd8fb2348389fdcb45fb37e42 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 28 Mar 2022 15:06:00 +0000 Subject: [PATCH 265/293] Start trying to hijack the cli projects table for the front end --- dds_web/templates/user/projects.html | 5 +++++ dds_web/web/user.py | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/dds_web/templates/user/projects.html b/dds_web/templates/user/projects.html index 9f057bd97..6b8d5dde6 100644 --- a/dds_web/templates/user/projects.html +++ b/dds_web/templates/user/projects.html @@ -6,5 +6,10 @@ {% block body %} +
      + {% for project in projects %} +
    • {{ project }}
      • + {% endfor %} +
    {% endblock %} \ No newline at end of file diff --git a/dds_web/web/user.py b/dds_web/web/user.py index d33cdcabc..7ac32c024 100644 --- a/dds_web/web/user.py +++ b/dds_web/web/user.py @@ -21,6 +21,7 @@ from dds_web.api import db_tools from dds_web.api.dds_decorators import logging_bind_request from dds_web.api.schemas import user_schemas +from dds_web.api.project import UserProjects from dds_web.api.user import DeleteUser from dds_web.database import models from dds_web.security.project_user_keys import update_user_keys_for_password_change @@ -567,5 +568,7 @@ def account_info(): @logging_bind_request def projects_info(): """User projects page""" + projects = UserProjects() - return flask.render_template("user/projects.html", enumerate=enumerate) \ No newline at end of file + + return flask.render_template("user/projects.html", projects = projects.get(), enumerate=enumerate) \ No newline at end of file From 190390f9d5b778ba3c8ee0dccadcdd3a5a50c825 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 28 Mar 2022 19:34:07 +0000 Subject: [PATCH 266/293] Open a second terminal window for using the dds cli --- .gitpod.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitpod.yml b/.gitpod.yml index 079a809df..17cdd9a76 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -12,6 +12,12 @@ tasks: chmod a+x /workspace/ && docker compose build --pull command: docker compose --profile cli up + - name: Open dds cli + openMode: split-right + command: > + gp await-port 5000 && + echo -e "\033[1;31mUse the dds cli in this terminal window\033[0m\n\033[0;33me.g.: dds auth login\033[0m" && + docker exec -it dds_cli bash ports: - port: 5000 # backend From 3be8529700f771dce1c0de15b63682f5caf1c631 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 28 Mar 2022 21:15:32 +0000 Subject: [PATCH 267/293] Give up trying to have multiple web tabs open --- .gitpod.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index 17cdd9a76..9ab45e43e 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -21,10 +21,12 @@ tasks: ports: - port: 5000 # backend - onOpen: open-browser + onOpen: open-preview visibility: public - port: 1080 # mailcatcher - onOpen: open-preview + # Can't have more than one preview at once currently :( + # open-browser is blocked by Chrome pop-up blocker + onOpen: ignore visibility: public - port: 9000 # minio onOpen: ignore From 01d8d2f782cac8aa52512f5795ddda20b07590af Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 28 Mar 2022 22:56:40 +0000 Subject: [PATCH 268/293] Build basic front-end page to list projects --- dds_web/api/project.py | 6 ++++ dds_web/templates/user/projects.html | 41 ++++++++++++++++++++++++---- dds_web/web/user.py | 6 ++-- 3 files changed, 45 insertions(+), 8 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index b8018cb14..1f02ccaac 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -408,6 +408,12 @@ class UserProjects(flask_restful.Resource): def get(self): """Get info regarding all projects which user is involved in.""" current_user = auth.current_user() + return self.format_project_dict(current_user) + + def format_project_dict(self, current_user): + """Given a logged in user, fetch projects and return as dict. + + Also used by web/user.py projects_info()""" # TODO: Return different things depending on if unit or not all_projects = list() diff --git a/dds_web/templates/user/projects.html b/dds_web/templates/user/projects.html index 6b8d5dde6..1459c404f 100644 --- a/dds_web/templates/user/projects.html +++ b/dds_web/templates/user/projects.html @@ -6,10 +6,41 @@ {% block body %} -
      - {% for project in projects %} -
    • {{ project }}
      • +{% if not projects.project_info %} + +

      No projects found.

      + +{% else %} + +

      Total filesize across all projects: {{ projects.total_size }}

      + + + + + + + + + + + + + + + {% for project in projects.project_info %} + + + + + + + + + {% endfor %} - + +
      Project IDTitlePrinciple InvestigatorStatusLast updatedSizeAccess
      {{ project['Project ID'] }}{{ project['Title'] }}{{ project['PI'] }}{{ project['Status'] }}{{ project['Last updated'] }}{{ project['Size'] }}{{ "✅" if project['Access'] else "❌" }}
      + +{% endif %} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/dds_web/web/user.py b/dds_web/web/user.py index 7ac32c024..7646f85ec 100644 --- a/dds_web/web/user.py +++ b/dds_web/web/user.py @@ -568,7 +568,7 @@ def account_info(): @logging_bind_request def projects_info(): """User projects page""" - projects = UserProjects() + projects_obj = UserProjects() + projects = projects_obj.format_project_dict(flask_login.current_user) - - return flask.render_template("user/projects.html", projects = projects.get(), enumerate=enumerate) \ No newline at end of file + return flask.render_template("user/projects.html", projects=projects, enumerate=enumerate) From 3b46f86ccf6a4e21820078c61eaa567f9c1bde6b Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 28 Mar 2022 23:07:09 +0000 Subject: [PATCH 269/293] Add some more default VSCode extensions --- .gitpod.yml | 5 ++++- .vscode/settings.json | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index 9ab45e43e..128d466f5 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -41,7 +41,10 @@ ports: vscode: extensions: - ms-azuretools.vscode-docker - - esbenp.prettier-vscode + - ms-python.python + - esbenp.prettier-vscode # Linting and style checking + - eamodio.gitlens # Quickly glimpse into whom, why, and when a line or code block was changed + - Gruntfuggly.todo-tree # Display TODO and FIXME in a tree view in the activity bar github: prebuilds: diff --git a/.vscode/settings.json b/.vscode/settings.json index eeee976e3..437299d00 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,6 @@ { "editor.defaultFormatter": "esbenp.prettier-vscode", "editor.formatOnSave": true, - "workbench.startupEditor": "none" + "workbench.startupEditor": "none", + "python.formatting.provider": "black" } From 7e6abe209f4cce9ba6ae19088ed5eb0936ec06a4 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 28 Mar 2022 14:26:49 +0000 Subject: [PATCH 270/293] Minor style improvements to 2fa email --- dds_web/templates/mail/authenticate.html | 25 ++++++++++++------------ 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/dds_web/templates/mail/authenticate.html b/dds_web/templates/mail/authenticate.html index b0d6436b5..c59dd4923 100644 --- a/dds_web/templates/mail/authenticate.html +++ b/dds_web/templates/mail/authenticate.html @@ -1,25 +1,24 @@ {% extends 'mail/mail_base.html' %} {% block body %} - +
      - Logo +

      + Logo +

      -

      - Here is the issued authentication code for SciLifeLab Data Delivery System (DDS) +

      + Here is your two-factor authentication code for the SciLifeLab Data Delivery System. +

      +

      + Please, copy and paste the code below in order to log in: +

      +

      + {{one_time_value}}

      -

      - Please, copy and paste the code below where you attempt to authenticate:

      -

      - {{one_time_value}}
      From e81eb34d45ea9c3f405c6ab8c7e39742e5fde733 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 28 Mar 2022 14:41:42 +0000 Subject: [PATCH 271/293] Tidy user profile page --- dds_web/__init__.py | 4 +++ dds_web/templates/user/account.html | 38 +++++++++-------------------- 2 files changed, 16 insertions(+), 26 deletions(-) diff --git a/dds_web/__init__.py b/dds_web/__init__.py index 0a80c8f64..68ed2c091 100644 --- a/dds_web/__init__.py +++ b/dds_web/__init__.py @@ -185,12 +185,16 @@ def create_app(testing=False, database_uri=None): def prepare(): """Populate flask globals for template rendering""" flask.g.current_user = None + flask.g.current_user_emails = None if auth.current_user(): flask.g.current_user = auth.current_user().username + flask.g.current_user_emails = auth.current_user().emails elif flask_login.current_user.is_authenticated: flask.g.current_user = flask_login.current_user.username + flask.g.current_user_emails = flask_login.current_user.emails elif flask.request.authorization: flask.g.current_user = flask.request.authorization.get("username") + flask.g.current_user_emails = flask.request.authorization.get("emails") # Setup logging handlers setup_logging(app) diff --git a/dds_web/templates/user/account.html b/dds_web/templates/user/account.html index c06216553..b6cd8f97a 100644 --- a/dds_web/templates/user/account.html +++ b/dds_web/templates/user/account.html @@ -8,51 +8,37 @@
      -
      - Username +
      + Username
      {{ g.current_user }}
      -
      - Password +
      + Password
      Change password
      -
      -
      - Name -
      -
      - {{ account_info.first_name }} {{ account_info.last_name }} -
      -
      -
      -
      - Permissions -
      -
      - {{ account_info.permissions }} -
      -
      - {% for i, email in enumerate(account_info.emails) %} + + {% for i, email in enumerate(g.current_user_emails) %}
      -
      +
      {% if i == 0 %} - Email + Email address {% endif %}
      - {{ email.address }} - {% if email.primary == true and len(account_info.emails) > 1 %} - Primary + {{ email.email }} + {% if email.primary == true %} + Primary {% endif %}
      {% endfor %} +
      {% endblock %} From 77aa1f44354d39e5afc422120857402b11fecd24 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 28 Mar 2022 14:49:37 +0000 Subject: [PATCH 272/293] Remove unused account_info dict --- dds_web/web/user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/web/user.py b/dds_web/web/user.py index 7cc97dd32..2c7e43864 100644 --- a/dds_web/web/user.py +++ b/dds_web/web/user.py @@ -559,4 +559,4 @@ def confirm_self_deletion(token): def account_info(): """User account page""" - return flask.render_template("user/account.html", account_info={}, enumerate=enumerate) + return flask.render_template("user/account.html", enumerate=enumerate) From a88da12b1d8701fb4977b766a7cc04104f45e9f6 Mon Sep 17 00:00:00 2001 From: Ina Date: Wed, 30 Mar 2022 10:05:11 +0200 Subject: [PATCH 273/293] apscheduler --- dds_web/__init__.py | 9 +++++-- dds_web/scheduled_tasks.py | 24 +++++++++++++++++++ dds_web/utils.py | 48 -------------------------------------- requirements.txt | 1 + 4 files changed, 32 insertions(+), 50 deletions(-) create mode 100644 dds_web/scheduled_tasks.py diff --git a/dds_web/__init__.py b/dds_web/__init__.py index 68ed2c091..f308ddeab 100644 --- a/dds_web/__init__.py +++ b/dds_web/__init__.py @@ -21,6 +21,7 @@ import flask_mail import flask_login import flask_migrate +from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore # import flask_qrcode from werkzeug.middleware.proxy_fix import ProxyFix @@ -30,6 +31,8 @@ import structlog import werkzeug +from dds_web.scheduled_tasks import scheduler + #################################################################################################### # GLOBAL VARIABLES ############################################################## GLOBAL VARIABLES # @@ -270,8 +273,10 @@ def load_user(user_id): app.register_blueprint(pages, url_prefix="") app.register_blueprint(auth_blueprint, url_prefix="") - # Set-up the schedulers - dds_web.utils.scheduler_wrapper() + # Set-up the scheduler + app.config["SCHEDULER_JOBSTORES"] = {"default": SQLAlchemyJobStore(engine=db.engine)} + scheduler.init_app(app) + scheduler.start() ENCRYPTION_KEY_BIT_LENGTH = 256 ENCRYPTION_KEY_CHAR_LENGTH = int(ENCRYPTION_KEY_BIT_LENGTH / 8) diff --git a/dds_web/scheduled_tasks.py b/dds_web/scheduled_tasks.py new file mode 100644 index 000000000..cda18faab --- /dev/null +++ b/dds_web/scheduled_tasks.py @@ -0,0 +1,24 @@ +import flask_apscheduler +import flask + +# Apscheduler +scheduler = flask_apscheduler.APScheduler() + + +@scheduler.task("interval", id="available_to_expired", seconds=10, misfire_grace_time=1) +def change_status_to_expired(): + print("Task: Change project status from Available to expired.", flush=True) + from dds_web.database import models + + # from dds_web.utils import current_time, page_query + + with scheduler.app.app_context(): + scheduler.app.logger.debug("This means that the app context works!") + test = models.Project.query.all() + print(test, flush=True) + + # for project in page_query( + # models.ProjectStatuses.query.filter(models.ProjectStatuses.deadline <= current_time()) + # ): + + # flask.current_app.logger.debug("Project: %s - Expires: %s", project, project.expires) \ No newline at end of file diff --git a/dds_web/utils.py b/dds_web/utils.py index b06b6ca5a..59f62d2d3 100644 --- a/dds_web/utils.py +++ b/dds_web/utils.py @@ -367,51 +367,3 @@ def create_one_time_password_email(user, hotp_value): return msg -#################################################################################################### -# BACKGROUND SCHEDULER ###################################################### BACKGROUND SCHEDULER # -#################################################################################################### - - -def scheduler_wrapper(): - - # Flask in debug mode spawns a child process so that it can restart the process each time the code changes, - # the new child process initializes and starts a new APScheduler, causing the jobs to be created twice - # within in the same database table: - # pymysql.err.IntegrityError: (1062, "Duplicate entry 'calc_costs' for key 'PRIMARY'") error - - # Apparently, the reload is done with a subprocess.call, so we have 2 different Python interpreters running at the same time! - # This also means that any if statement or replace_existing=FALSE paramenter in add_job() won't prevent these errors. - # This if statement hopefully solves the issue: - - if flask.helpers.get_debug_flag() and not werkzeug.serving.is_running_from_reloader(): - return - - scheduler = background.BackgroundScheduler( - { - "apscheduler.jobstores.default": { - "type": "sqlalchemy", - # "url": flask.current_app.config.get("SQLALCHEMY_DATABASE_URI"), - "engine": db.engine, - }, - "apscheduler.timezone": "Europe/Stockholm", - } - ) - - scheduler.start() - flask.current_app.logger.info("Started main scheduler") - - # Extract all jobIDs currently scheduled - joblist = scheduler.get_jobs() - jobid = [] - for job in joblist: - job_id = getattr(job, "id") - jobid.append(job_id) - - # Shut down the scheduler when exiting the app - atexit.register(scheduler.shutdown) - - # Print the currently scheduled jobs as verification: - joblist = scheduler.get_jobs() - flask.current_app.logger.info("Currently scheduled jobs:") - for job in joblist: - flask.current_app.logger.info(f"Job: {job}") diff --git a/requirements.txt b/requirements.txt index 6b1b72467..18534fd67 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,6 +19,7 @@ email-validator==1.1.3 factory-boy==3.2.1 Faker==12.0.0 Flask==2.0.2 +Flask-APScheduler==1.12.3 Flask-HTTPAuth==4.5.0 Flask-Limiter==2.1.3 Flask-Login==0.5.0 From 0f96c2a9947d8e28de635b473145c9ddc38bb2c7 Mon Sep 17 00:00:00 2001 From: Ina Date: Wed, 30 Mar 2022 10:07:05 +0200 Subject: [PATCH 274/293] black --- dds_web/scheduled_tasks.py | 2 +- dds_web/utils.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/dds_web/scheduled_tasks.py b/dds_web/scheduled_tasks.py index cda18faab..392167e02 100644 --- a/dds_web/scheduled_tasks.py +++ b/dds_web/scheduled_tasks.py @@ -21,4 +21,4 @@ def change_status_to_expired(): # models.ProjectStatuses.query.filter(models.ProjectStatuses.deadline <= current_time()) # ): - # flask.current_app.logger.debug("Project: %s - Expires: %s", project, project.expires) \ No newline at end of file + # flask.current_app.logger.debug("Project: %s - Expires: %s", project, project.expires) diff --git a/dds_web/utils.py b/dds_web/utils.py index 59f62d2d3..39e91ae35 100644 --- a/dds_web/utils.py +++ b/dds_web/utils.py @@ -365,5 +365,3 @@ def create_one_time_password_email(user, hotp_value): ) return msg - - From 2b34436ffc3cf5a0eb54c432c3095d67c416bf53 Mon Sep 17 00:00:00 2001 From: Ina Date: Wed, 30 Mar 2022 10:26:05 +0200 Subject: [PATCH 275/293] don't print everything --- dds_web/scheduled_tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dds_web/scheduled_tasks.py b/dds_web/scheduled_tasks.py index 392167e02..a6724c486 100644 --- a/dds_web/scheduled_tasks.py +++ b/dds_web/scheduled_tasks.py @@ -7,15 +7,15 @@ @scheduler.task("interval", id="available_to_expired", seconds=10, misfire_grace_time=1) def change_status_to_expired(): - print("Task: Change project status from Available to expired.", flush=True) + # print("Task: Change project status from Available to expired.", flush=True) from dds_web.database import models # from dds_web.utils import current_time, page_query with scheduler.app.app_context(): - scheduler.app.logger.debug("This means that the app context works!") + # scheduler.app.logger.debug("This means that the app context works!") test = models.Project.query.all() - print(test, flush=True) + # print(test, flush=True) # for project in page_query( # models.ProjectStatuses.query.filter(models.ProjectStatuses.deadline <= current_time()) From a8e1c64ebefe141f4ca9eefb4bcebb8b1b4af89b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Wed, 30 Mar 2022 11:28:29 +0200 Subject: [PATCH 276/293] Update dds_web/api/project.py --- dds_web/api/project.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 1f02ccaac..4d23297ab 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -407,8 +407,7 @@ class UserProjects(flask_restful.Resource): @logging_bind_request def get(self): """Get info regarding all projects which user is involved in.""" - current_user = auth.current_user() - return self.format_project_dict(current_user) + return self.format_project_dict(current_user=auth.current_user()) def format_project_dict(self, current_user): """Given a logged in user, fetch projects and return as dict. From 07a345a785d84488dc00d76545076f3f5892d1d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Wed, 30 Mar 2022 11:28:34 +0200 Subject: [PATCH 277/293] Update dds_web/api/project.py --- dds_web/api/project.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index 4d23297ab..e54454256 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -412,8 +412,8 @@ def get(self): def format_project_dict(self, current_user): """Given a logged in user, fetch projects and return as dict. - Also used by web/user.py projects_info()""" - + Also used by web/user.py projects_info() + """ # TODO: Return different things depending on if unit or not all_projects = list() From d9b8e73ef358f5990b4d7ef5e4db7d06eaee42d2 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 30 Mar 2022 09:45:28 +0000 Subject: [PATCH 278/293] Remove the Gitlens extension from gitpod It's really spammy on load in the web editor and not really needed. --- .gitpod.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index 128d466f5..27b6c251a 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -43,7 +43,6 @@ vscode: - ms-azuretools.vscode-docker - ms-python.python - esbenp.prettier-vscode # Linting and style checking - - eamodio.gitlens # Quickly glimpse into whom, why, and when a line or code block was changed - Gruntfuggly.todo-tree # Display TODO and FIXME in a tree view in the activity bar github: From c4ff78e2c441cbd116da10fdf970509092d9df28 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 30 Mar 2022 10:06:53 +0000 Subject: [PATCH 279/293] Add link to installation docs Added navbar link to the installation docs. Also made the icons in the navbar heavier weight to stand out a little more. --- dds_web/templates/navbar.html | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/dds_web/templates/navbar.html b/dds_web/templates/navbar.html index 2d0d55a45..ba408b414 100644 --- a/dds_web/templates/navbar.html +++ b/dds_web/templates/navbar.html @@ -9,39 +9,45 @@
      • - + Home
      • - - + + + Download DDS + +
        • +
      • + + Documentation
        • {% if g.current_user %}
      • - + Projects
      • - + {{ g.current_user }}
      • - + Logout
        • {% else %}
      • - + Log In
        • From 2d7ae2599cab96c0da1047121d235f9bcf64e1d1 Mon Sep 17 00:00:00 2001 From: Zishan Mirza Date: Wed, 30 Mar 2022 10:42:33 +0000 Subject: [PATCH 280/293] Project Access Reset and Send Email This patch sends an email to all Unit Admins when a Unit Admin has reset their password. Signed-off-by: Zishan Mirza --- CHANGELOG.md | 4 ++ .../templates/mail/project_access_reset.html | 23 +++++++ .../templates/mail/project_access_reset.txt | 5 ++ dds_web/utils.py | 24 ++++++++ dds_web/web/user.py | 47 +++++++++++--- tests/test_user_reset_password.py | 61 +++++++++++++++++++ 6 files changed, 156 insertions(+), 8 deletions(-) create mode 100644 dds_web/templates/mail/project_access_reset.html create mode 100644 dds_web/templates/mail/project_access_reset.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 96e6d6b69..491b2827c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,3 +61,7 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - Let project description include . and , ([#1080](https://github.com/ScilifelabDataCentre/dds_web/pull/1080)) - Catch OperationalError if there is a database malfunction in `files.py` ([#1089](https://github.com/ScilifelabDataCentre/dds_web/pull/1089)) - Switched the validation for the principal investigator from string to email ([#1084](https://github.com/ScilifelabDataCentre/dds_web/pull/1084)). + +## Sprint (2022-03-23 - 2022-04-06) + +- Send an email to all Unit Admins when a Unit Admin has reset their password ([#1110](https://github.com/ScilifelabDataCentre/dds_web/pull/1110)). diff --git a/dds_web/templates/mail/project_access_reset.html b/dds_web/templates/mail/project_access_reset.html new file mode 100644 index 000000000..a4e01d017 --- /dev/null +++ b/dds_web/templates/mail/project_access_reset.html @@ -0,0 +1,23 @@ +{% extends 'mail/mail_base.html' %} +{% block body %} + + + + + + + +
        + Logo +
        +

        + Important! The Unit Admin, {{email}}, has recently reset their password and therefore lost access to all projects within your + unit. To reduce the risk of future data loss, please grant them access again with the following command: +

        +

        + dds project access fix {{email}} +

        +
        +{% endblock %} diff --git a/dds_web/templates/mail/project_access_reset.txt b/dds_web/templates/mail/project_access_reset.txt new file mode 100644 index 000000000..75004b142 --- /dev/null +++ b/dds_web/templates/mail/project_access_reset.txt @@ -0,0 +1,5 @@ +Important! The Unit Admin, {{email}}, has recently reset their password and therefore +lost access to all projects within your unit. To reduce the risk of future data loss, +please grant them access again with the following command: + + dds project access fix {{email}} diff --git a/dds_web/utils.py b/dds_web/utils.py index b06b6ca5a..089188568 100644 --- a/dds_web/utils.py +++ b/dds_web/utils.py @@ -277,6 +277,30 @@ def send_reset_email(email_row, token): mail.send(msg) +def send_project_access_reset_email(email_row, email, token): + """Generate password reset email.""" + msg = flask_mail.Message( + "WARNING! A Unit Admin has lost access", + recipients=[email_row.email], + ) + + # Need to attach the image to be able to use it + msg.attach( + "scilifelab_logo.png", + "image/png", + open(os.path.join(flask.current_app.static_folder, "img/scilifelab_logo.png"), "rb").read(), + "inline", + headers=[ + ["Content-ID", ""], + ], + ) + + msg.body = flask.render_template("mail/project_access_reset.txt", email=email) + msg.html = flask.render_template("mail/project_access_reset.html", email=email) + + mail.send(msg) + + def is_safe_url(target): """Check if the url is safe for redirects.""" ref_url = urllib.parse.urlparse(flask.request.host_url) diff --git a/dds_web/web/user.py b/dds_web/web/user.py index 2c7e43864..e8073efb0 100644 --- a/dds_web/web/user.py +++ b/dds_web/web/user.py @@ -448,17 +448,48 @@ def password_reset_completed(): return flask.redirect(flask.url_for("auth_blueprint.index")) units_to_contact = {} + unit_admins_to_contact = {} + if user.role != "Super Admin": for project in user.projects: - if project.responsible_unit.external_display_name not in units_to_contact: - units_to_contact[ - project.responsible_unit.external_display_name - ] = project.responsible_unit.contact_email - return flask.render_template( - "user/password_reset_completed.html", units_to_contact=units_to_contact - ) + if user.role == "Unit Admin": + users = ( + db.session.query(models.User) + .join(models.UnitUser) + .join(models.Email) + .with_entities( + models.User.username, + models.User.name, + models.UnitUser.unit_id, + models.UnitUser.is_admin, + models.Email.email, + ) + .filter(models.UnitUser.unit_id == user.unit_id) + .filter(models.UnitUser.is_admin == True) + .filter(models.User.username != user.username) + .all() + ) + email = ( + db.session.query(models.Email) + .with_entities(models.Email.email) + .filter(models.Email.user_id == user.username) + .first() + ) - return flask.render_template("user/password_reset_completed.html") + unit_admins_to_contact = users + else: + if project.responsible_unit.external_display_name not in units_to_contact: + units_to_contact[ + project.responsible_unit.external_display_name + ] = project.responsible_unit.contact_email + + if len(unit_admins_to_contact) > 0: + for unit_admin in unit_admins_to_contact: + dds_web.utils.send_project_access_reset_email(unit_admin, email[0], token) + + return flask.render_template( + "user/password_reset_completed.html", units_to_contact=units_to_contact + ) @auth_blueprint.route("/change_password", methods=["GET", "POST"]) diff --git a/tests/test_user_reset_password.py b/tests/test_user_reset_password.py index 537ca51eb..18e760537 100644 --- a/tests/test_user_reset_password.py +++ b/tests/test_user_reset_password.py @@ -477,3 +477,64 @@ def test_reset_password_unituser(client): # Check that public key has changed user_public_key_after = user.public_key assert user_public_key_before != user_public_key_after + + +def test_reset_password_unitadmin(client): + user = models.User.query.filter_by(username="unitadmin").first() + nr_proj_user_keys_total_before = models.ProjectUserKeys.query.count() + assert nr_proj_user_keys_total_before > 0 + + nr_proj_user_keys_before = len(user.project_user_keys) + assert nr_proj_user_keys_before > 0 + + user_pw_hash_before = user._password_hash + user_public_key_before = user.public_key + + # Add new row to password reset + new_reset_row = models.PasswordReset( + user=user, email=user.primary_email, issued=utils.timestamp() + ) + db.session.add(new_reset_row) + db.session.commit() + + # Need to use a valid token for the get request to get the form token + valid_reset_token = get_valid_reset_token("unitadmin") + response = client.get( + tests.DDSEndpoint.RESET_PASSWORD + valid_reset_token, follow_redirects=True + ) + + assert response.status_code == http.HTTPStatus.OK + assert flask.request.path == tests.DDSEndpoint.RESET_PASSWORD + valid_reset_token + + form_token = flask.g.csrf_token + form_data = { + "csrf_token": form_token, + "password": "NewPassword123", + "confirm_password": "NewPassword123", + "submit": "Reset Password", + } + + response = client.post( + tests.DDSEndpoint.RESET_PASSWORD + valid_reset_token, json=form_data, follow_redirects=True + ) + assert response.status_code == http.HTTPStatus.OK + assert flask.request.path == tests.DDSEndpoint.PASSWORD_RESET_COMPLETED + + user = models.User.query.filter_by(username="unitadmin").first() + + # All users project keys should have been removed + nr_proj_user_keys_after = len(user.project_user_keys) + assert nr_proj_user_keys_after == 0 + + # Total nr of project user keys should be decreased + nr_proj_user_keys_total_after = models.ProjectUserKeys.query.count() + assert nr_proj_user_keys_total_after < nr_proj_user_keys_total_before + assert nr_proj_user_keys_total_after != nr_proj_user_keys_total_before + + # Password should have changed + user_pw_hash_after = user._password_hash + assert user_pw_hash_before != user_pw_hash_after + + # Check that public key has changed + user_public_key_after = user.public_key + assert user_public_key_before != user_public_key_after From adc74b922a8e344ecf05bb9f36bab821461be801 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 30 Mar 2022 13:45:22 +0200 Subject: [PATCH 281/293] Changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96e6d6b69..14ee7b9c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,3 +61,7 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - Let project description include . and , ([#1080](https://github.com/ScilifelabDataCentre/dds_web/pull/1080)) - Catch OperationalError if there is a database malfunction in `files.py` ([#1089](https://github.com/ScilifelabDataCentre/dds_web/pull/1089)) - Switched the validation for the principal investigator from string to email ([#1084](https://github.com/ScilifelabDataCentre/dds_web/pull/1084)). + +## Sprint (2022-03-22 - 2022-04-06) + +- Add link in navbar to the installation documentation ([#1112](https://github.com/ScilifelabDataCentre/dds_web/pull/1112)) \ No newline at end of file From 3a105427df04a6e1853c273c565c1b11b1623d9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Wed, 30 Mar 2022 13:49:46 +0200 Subject: [PATCH 282/293] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14ee7b9c3..947be5ef5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,4 +64,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe ## Sprint (2022-03-22 - 2022-04-06) -- Add link in navbar to the installation documentation ([#1112](https://github.com/ScilifelabDataCentre/dds_web/pull/1112)) \ No newline at end of file +- Add link in navbar to the installation documentation ([#1112](https://github.com/ScilifelabDataCentre/dds_web/pull/1112)) From 11c970e7b0340429743c185176b7ec6fa0960d45 Mon Sep 17 00:00:00 2001 From: Ina Date: Wed, 30 Mar 2022 10:05:11 +0200 Subject: [PATCH 283/293] apscheduler --- dds_web/__init__.py | 9 +++++-- dds_web/scheduled_tasks.py | 24 +++++++++++++++++++ dds_web/utils.py | 48 -------------------------------------- requirements.txt | 1 + 4 files changed, 32 insertions(+), 50 deletions(-) create mode 100644 dds_web/scheduled_tasks.py diff --git a/dds_web/__init__.py b/dds_web/__init__.py index 68ed2c091..f308ddeab 100644 --- a/dds_web/__init__.py +++ b/dds_web/__init__.py @@ -21,6 +21,7 @@ import flask_mail import flask_login import flask_migrate +from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore # import flask_qrcode from werkzeug.middleware.proxy_fix import ProxyFix @@ -30,6 +31,8 @@ import structlog import werkzeug +from dds_web.scheduled_tasks import scheduler + #################################################################################################### # GLOBAL VARIABLES ############################################################## GLOBAL VARIABLES # @@ -270,8 +273,10 @@ def load_user(user_id): app.register_blueprint(pages, url_prefix="") app.register_blueprint(auth_blueprint, url_prefix="") - # Set-up the schedulers - dds_web.utils.scheduler_wrapper() + # Set-up the scheduler + app.config["SCHEDULER_JOBSTORES"] = {"default": SQLAlchemyJobStore(engine=db.engine)} + scheduler.init_app(app) + scheduler.start() ENCRYPTION_KEY_BIT_LENGTH = 256 ENCRYPTION_KEY_CHAR_LENGTH = int(ENCRYPTION_KEY_BIT_LENGTH / 8) diff --git a/dds_web/scheduled_tasks.py b/dds_web/scheduled_tasks.py new file mode 100644 index 000000000..cda18faab --- /dev/null +++ b/dds_web/scheduled_tasks.py @@ -0,0 +1,24 @@ +import flask_apscheduler +import flask + +# Apscheduler +scheduler = flask_apscheduler.APScheduler() + + +@scheduler.task("interval", id="available_to_expired", seconds=10, misfire_grace_time=1) +def change_status_to_expired(): + print("Task: Change project status from Available to expired.", flush=True) + from dds_web.database import models + + # from dds_web.utils import current_time, page_query + + with scheduler.app.app_context(): + scheduler.app.logger.debug("This means that the app context works!") + test = models.Project.query.all() + print(test, flush=True) + + # for project in page_query( + # models.ProjectStatuses.query.filter(models.ProjectStatuses.deadline <= current_time()) + # ): + + # flask.current_app.logger.debug("Project: %s - Expires: %s", project, project.expires) \ No newline at end of file diff --git a/dds_web/utils.py b/dds_web/utils.py index b06b6ca5a..59f62d2d3 100644 --- a/dds_web/utils.py +++ b/dds_web/utils.py @@ -367,51 +367,3 @@ def create_one_time_password_email(user, hotp_value): return msg -#################################################################################################### -# BACKGROUND SCHEDULER ###################################################### BACKGROUND SCHEDULER # -#################################################################################################### - - -def scheduler_wrapper(): - - # Flask in debug mode spawns a child process so that it can restart the process each time the code changes, - # the new child process initializes and starts a new APScheduler, causing the jobs to be created twice - # within in the same database table: - # pymysql.err.IntegrityError: (1062, "Duplicate entry 'calc_costs' for key 'PRIMARY'") error - - # Apparently, the reload is done with a subprocess.call, so we have 2 different Python interpreters running at the same time! - # This also means that any if statement or replace_existing=FALSE paramenter in add_job() won't prevent these errors. - # This if statement hopefully solves the issue: - - if flask.helpers.get_debug_flag() and not werkzeug.serving.is_running_from_reloader(): - return - - scheduler = background.BackgroundScheduler( - { - "apscheduler.jobstores.default": { - "type": "sqlalchemy", - # "url": flask.current_app.config.get("SQLALCHEMY_DATABASE_URI"), - "engine": db.engine, - }, - "apscheduler.timezone": "Europe/Stockholm", - } - ) - - scheduler.start() - flask.current_app.logger.info("Started main scheduler") - - # Extract all jobIDs currently scheduled - joblist = scheduler.get_jobs() - jobid = [] - for job in joblist: - job_id = getattr(job, "id") - jobid.append(job_id) - - # Shut down the scheduler when exiting the app - atexit.register(scheduler.shutdown) - - # Print the currently scheduled jobs as verification: - joblist = scheduler.get_jobs() - flask.current_app.logger.info("Currently scheduled jobs:") - for job in joblist: - flask.current_app.logger.info(f"Job: {job}") diff --git a/requirements.txt b/requirements.txt index 6b1b72467..18534fd67 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,6 +19,7 @@ email-validator==1.1.3 factory-boy==3.2.1 Faker==12.0.0 Flask==2.0.2 +Flask-APScheduler==1.12.3 Flask-HTTPAuth==4.5.0 Flask-Limiter==2.1.3 Flask-Login==0.5.0 From 0646ac6c694d219e25201791f854a0aceb762025 Mon Sep 17 00:00:00 2001 From: Ina Date: Wed, 30 Mar 2022 10:07:05 +0200 Subject: [PATCH 284/293] black --- dds_web/scheduled_tasks.py | 2 +- dds_web/utils.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/dds_web/scheduled_tasks.py b/dds_web/scheduled_tasks.py index cda18faab..392167e02 100644 --- a/dds_web/scheduled_tasks.py +++ b/dds_web/scheduled_tasks.py @@ -21,4 +21,4 @@ def change_status_to_expired(): # models.ProjectStatuses.query.filter(models.ProjectStatuses.deadline <= current_time()) # ): - # flask.current_app.logger.debug("Project: %s - Expires: %s", project, project.expires) \ No newline at end of file + # flask.current_app.logger.debug("Project: %s - Expires: %s", project, project.expires) diff --git a/dds_web/utils.py b/dds_web/utils.py index 59f62d2d3..39e91ae35 100644 --- a/dds_web/utils.py +++ b/dds_web/utils.py @@ -365,5 +365,3 @@ def create_one_time_password_email(user, hotp_value): ) return msg - - From aa26c0cec230d676e6c125626e25367eaeb45904 Mon Sep 17 00:00:00 2001 From: Ina Date: Wed, 30 Mar 2022 10:26:05 +0200 Subject: [PATCH 285/293] don't print everything --- dds_web/scheduled_tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dds_web/scheduled_tasks.py b/dds_web/scheduled_tasks.py index 392167e02..a6724c486 100644 --- a/dds_web/scheduled_tasks.py +++ b/dds_web/scheduled_tasks.py @@ -7,15 +7,15 @@ @scheduler.task("interval", id="available_to_expired", seconds=10, misfire_grace_time=1) def change_status_to_expired(): - print("Task: Change project status from Available to expired.", flush=True) + # print("Task: Change project status from Available to expired.", flush=True) from dds_web.database import models # from dds_web.utils import current_time, page_query with scheduler.app.app_context(): - scheduler.app.logger.debug("This means that the app context works!") + # scheduler.app.logger.debug("This means that the app context works!") test = models.Project.query.all() - print(test, flush=True) + # print(test, flush=True) # for project in page_query( # models.ProjectStatuses.query.filter(models.ProjectStatuses.deadline <= current_time()) From aab2d26bd4c53515dff7a487850bc38749978457 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Wed, 30 Mar 2022 13:06:06 +0200 Subject: [PATCH 286/293] Update requirements.txt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Linus Östberg --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 18534fd67..51b5826f6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ email-validator==1.1.3 factory-boy==3.2.1 Faker==12.0.0 Flask==2.0.2 -Flask-APScheduler==1.12.3 +Flask-APScheduler==1.12.1 Flask-HTTPAuth==4.5.0 Flask-Limiter==2.1.3 Flask-Login==0.5.0 From 8a23aca2cdc0cab229d027dae29ea7ad7d98cdaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ina=20Od=C3=A9n=20=C3=96sterbo?= <35953392+inaod568@users.noreply.github.com> Date: Wed, 30 Mar 2022 13:47:58 +0200 Subject: [PATCH 287/293] Update requirements.txt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Linus Östberg --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 51b5826f6..18534fd67 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ email-validator==1.1.3 factory-boy==3.2.1 Faker==12.0.0 Flask==2.0.2 -Flask-APScheduler==1.12.1 +Flask-APScheduler==1.12.3 Flask-HTTPAuth==4.5.0 Flask-Limiter==2.1.3 Flask-Login==0.5.0 From 2f4f5fd286011190ea828e49321c852172a504b7 Mon Sep 17 00:00:00 2001 From: Ina Date: Wed, 30 Mar 2022 15:12:28 +0200 Subject: [PATCH 288/293] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 947be5ef5..e6efc2a8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,3 +65,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe ## Sprint (2022-03-22 - 2022-04-06) - Add link in navbar to the installation documentation ([#1112](https://github.com/ScilifelabDataCentre/dds_web/pull/1112)) +- Change from apscheduler to flask-apscheduler - solves the app context issue ([#1109](https://github.com/ScilifelabDataCentre/dds_web/pull/1109)) \ No newline at end of file From 7882ca22f6a50b523092763b182bddcb4e360211 Mon Sep 17 00:00:00 2001 From: Ina Date: Wed, 30 Mar 2022 15:14:46 +0200 Subject: [PATCH 289/293] last line in changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6efc2a8b..8a0da35ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,4 +65,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe ## Sprint (2022-03-22 - 2022-04-06) - Add link in navbar to the installation documentation ([#1112](https://github.com/ScilifelabDataCentre/dds_web/pull/1112)) -- Change from apscheduler to flask-apscheduler - solves the app context issue ([#1109](https://github.com/ScilifelabDataCentre/dds_web/pull/1109)) \ No newline at end of file +- Change from apscheduler to flask-apscheduler - solves the app context issue ([#1109](https://github.com/ScilifelabDataCentre/dds_web/pull/1109)) From 1e819ea7f0fda3474846ff2096b963a1301a67a2 Mon Sep 17 00:00:00 2001 From: Ina Date: Wed, 30 Mar 2022 15:53:17 +0200 Subject: [PATCH 290/293] trying to trigger the other tests --- dds_web/web/user.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dds_web/web/user.py b/dds_web/web/user.py index 45bfa55cb..76d19f3ad 100644 --- a/dds_web/web/user.py +++ b/dds_web/web/user.py @@ -166,8 +166,7 @@ def cancel_2fa(): error_message=ddserr.TooManyRequestsError.description, ) def confirm_2fa(): - """Finalize login by validating the authentication one-time token""" - + """Finalize login by validating the authentication one-time token.""" # Redirect to index if user is already authenticated if flask_login.current_user.is_authenticated: return flask.redirect(flask.url_for("pages.home")) From 0aeeaf9e08c95a649bf5664709160f7b6fa0bf83 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Mar 2022 14:12:51 +0000 Subject: [PATCH 291/293] Bump minimist from 1.2.5 to 1.2.6 in /dds_web/static Bumps [minimist](https://github.com/substack/minimist) from 1.2.5 to 1.2.6. - [Release notes](https://github.com/substack/minimist/releases) - [Commits](https://github.com/substack/minimist/compare/1.2.5...1.2.6) --- updated-dependencies: - dependency-name: minimist dependency-type: indirect ... Signed-off-by: dependabot[bot] --- dds_web/static/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dds_web/static/package-lock.json b/dds_web/static/package-lock.json index 267f6c80b..a35e8613c 100644 --- a/dds_web/static/package-lock.json +++ b/dds_web/static/package-lock.json @@ -3167,9 +3167,9 @@ } }, "node_modules/minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==", "dev": true }, "node_modules/minimist-options": { @@ -8566,9 +8566,9 @@ } }, "minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==", "dev": true }, "minimist-options": { From 6f7146cb6f9b5a12abc7ece321220a62a054fd16 Mon Sep 17 00:00:00 2001 From: Ina Date: Thu, 31 Mar 2022 16:41:30 +0200 Subject: [PATCH 292/293] unanswered inviteadded --- dds_web/api/project.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dds_web/api/project.py b/dds_web/api/project.py index e54454256..e5d25befb 100644 --- a/dds_web/api/project.py +++ b/dds_web/api/project.py @@ -661,6 +661,7 @@ def post(self): for user in p_info["users_to_add"]: try: existing_user = user_schemas.UserSchema().load(user) + unanswered_invite = user_schemas.UnansweredInvite().load(user) except ( marshmallow.exceptions.ValidationError, sqlalchemy.exc.OperationalError, @@ -673,7 +674,7 @@ def post(self): user_addition_statuses.append(addition_status) continue - if not existing_user: + if not existing_user and not unanswered_invite: # Send invite if the user doesn't exist invite_user_result = AddUser.invite_user( email=user.get("email"), @@ -694,7 +695,7 @@ def post(self): addition_status = "" try: add_user_result = AddUser.add_to_project( - whom=existing_user, + whom=existing_user or unanswered_invite, project=new_project, role=user.get("role"), ) From 80552ac2294a855de55d1e307946d4ac1905c42e Mon Sep 17 00:00:00 2001 From: Ina Date: Fri, 1 Apr 2022 07:14:20 +0200 Subject: [PATCH 293/293] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5af812a01..693e92890 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,3 +68,4 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe - Add link in navbar to the installation documentation ([#1112](https://github.com/ScilifelabDataCentre/dds_web/pull/1112)) - Change from apscheduler to flask-apscheduler - solves the app context issue ([#1109](https://github.com/ScilifelabDataCentre/dds_web/pull/1109)) - Send an email to all Unit Admins when a Unit Admin has reset their password ([#1110](https://github.com/ScilifelabDataCentre/dds_web/pull/1110)). +- Patch: Add check for unanswered invite when creating project and adding user who is already invited ([#1117](https://github.com/ScilifelabDataCentre/dds_web/pull/1117))