From 1340261fc70267b432b806f9771c00d02fcfa2b1 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 12 Sep 2024 13:51:12 +0200 Subject: [PATCH 01/59] send only to unitusers motd --- dds_web/api/superadmin_only.py | 16 +++++++++- dds_web/static/swagger.yaml | 3 ++ dds_web/static/swaggerv3.yaml | 3 ++ tests/api/test_superadmin_only.py | 36 ++++++++++++++++++++-- tests/tests_v3/api/test_superadmin_only.py | 36 ++++++++++++++++++++-- 5 files changed, 89 insertions(+), 5 deletions(-) diff --git a/dds_web/api/superadmin_only.py b/dds_web/api/superadmin_only.py index 6afc9b7e1..42be6e724 100644 --- a/dds_web/api/superadmin_only.py +++ b/dds_web/api/superadmin_only.py @@ -163,6 +163,20 @@ def post(self): if not motd_obj or not motd_obj.active: raise ddserr.DDSArgumentError(message=f"There is no active MOTD with ID '{motd_id}'.") + # check if sent to unit personnel only or all users + unit_personnel_only: bool = request_json.get("unit_personnel_only", False) + if not isinstance(unit_personnel_only, bool): + raise ddserr.DDSArgumentError( + message="The 'unit_personnel_only' argument must be a boolean." + ) + if unit_personnel_only: + unit_usernames = db.session.query(models.UnitUser.username) + users_to_send = db.session.query(models.User).filter( + models.User.username.in_(unit_usernames) + ) + else: + users_to_send = db.session.query(models.User) + # Create email content # put motd_obj.message etc in there etc subject: str = "Important Information: Data Delivery System" @@ -172,7 +186,7 @@ def post(self): # Setup email connection with mail.connect() as conn: # Email users - for user in utils.page_query(db.session.query(models.User)): + for user in utils.page_query(users_to_send): primary_email = user.primary_email if not primary_email: flask.current_app.logger.warning( diff --git a/dds_web/static/swagger.yaml b/dds_web/static/swagger.yaml index 5b5cb8686..e53f9696e 100644 --- a/dds_web/static/swagger.yaml +++ b/dds_web/static/swagger.yaml @@ -1447,6 +1447,9 @@ paths: motd_id: type: integer example: 1 + unit_personnel_only: + type: boolean + example: false /user/find: get: tags: diff --git a/dds_web/static/swaggerv3.yaml b/dds_web/static/swaggerv3.yaml index 7417afb91..dcd166491 100644 --- a/dds_web/static/swaggerv3.yaml +++ b/dds_web/static/swaggerv3.yaml @@ -1409,6 +1409,9 @@ paths: motd_id: type: integer example: 1 + unit_personnel_only: + type: boolean + example: false /user/find: get: tags: diff --git a/tests/api/test_superadmin_only.py b/tests/api/test_superadmin_only.py index ca1d5d428..9bf439be0 100644 --- a/tests/api/test_superadmin_only.py +++ b/tests/api/test_superadmin_only.py @@ -588,7 +588,7 @@ def test_send_motd_no_primary_email(client: flask.testing.FlaskClient) -> None: assert "incorrect subject" not in outbox[-1].subject -def test_send_motd_ok(client: flask.testing.FlaskClient) -> None: +def test_send_motd_ok_all(client: flask.testing.FlaskClient) -> None: """Send a motd to all users.""" # Authenticate token: typing.Dict = get_token(username=users["Super Admin"], client=client) @@ -609,7 +609,39 @@ def test_send_motd_ok(client: flask.testing.FlaskClient) -> None: # Attempt request and catch email with mail.record_messages() as outbox: response: werkzeug.test.WrapperTestResponse = client.post( - tests.DDSEndpoint.MOTD_SEND, headers=token, json={"motd_id": created_motd.id} + tests.DDSEndpoint.MOTD_SEND, + headers=token, + json={"motd_id": created_motd.id, "unit_personnel_only": False}, + ) + assert response.status_code == http.HTTPStatus.OK + assert len(outbox) == num_users + assert "Important Information: Data Delivery System" in outbox[-1].subject + + +def test_send_motd_ok_unitusers(client: flask.testing.FlaskClient) -> None: + """Send a motd to all unitusers users.""" + # Authenticate + token: typing.Dict = get_token(username=users["Super Admin"], client=client) + + # Create a motd + message: str = "This is a message that should become a MOTD and then be sent to all the users." + new_motd: models.MOTD = models.MOTD(message=message) + db.session.add(new_motd) + db.session.commit() + + # Make sure the motd is created + created_motd: models.MOTD = models.MOTD.query.filter_by(message=message).one_or_none() + assert created_motd + + # Get number of users + num_users = models.UnitUser.query.count() + + # Attempt request and catch email + with mail.record_messages() as outbox: + response: werkzeug.test.WrapperTestResponse = client.post( + tests.DDSEndpoint.MOTD_SEND, + headers=token, + json={"motd_id": created_motd.id, "unit_personnel_only": True}, ) assert response.status_code == http.HTTPStatus.OK assert len(outbox) == num_users diff --git a/tests/tests_v3/api/test_superadmin_only.py b/tests/tests_v3/api/test_superadmin_only.py index b70658819..43af9e7df 100644 --- a/tests/tests_v3/api/test_superadmin_only.py +++ b/tests/tests_v3/api/test_superadmin_only.py @@ -575,7 +575,7 @@ def test_send_motd_no_primary_email(client: flask.testing.FlaskClient) -> None: assert "incorrect subject" not in outbox[-1].subject -def test_send_motd_ok(client: flask.testing.FlaskClient) -> None: +def test_send_motd_ok_all(client: flask.testing.FlaskClient) -> None: """Send a motd to all users.""" # Authenticate token: typing.Dict = get_token(username=users["Super Admin"], client=client) @@ -596,7 +596,39 @@ def test_send_motd_ok(client: flask.testing.FlaskClient) -> None: # Attempt request and catch email with mail.record_messages() as outbox: response: werkzeug.test.WrapperTestResponse = client.post( - tests.DDSEndpoint.MOTD_SEND, headers=token, json={"motd_id": created_motd.id} + tests.DDSEndpoint.MOTD_SEND, + headers=token, + json={"motd_id": created_motd.id, "unit_personnel_only": False}, + ) + assert response.status_code == http.HTTPStatus.OK + assert len(outbox) == num_users + assert "Important Information: Data Delivery System" in outbox[-1].subject + + +def test_send_motd_ok_unitusers(client: flask.testing.FlaskClient) -> None: + """Send a motd to all unitusers users.""" + # Authenticate + token: typing.Dict = get_token(username=users["Super Admin"], client=client) + + # Create a motd + message: str = "This is a message that should become a MOTD and then be sent to all the users." + new_motd: models.MOTD = models.MOTD(message=message) + db.session.add(new_motd) + db.session.commit() + + # Make sure the motd is created + created_motd: models.MOTD = models.MOTD.query.filter_by(message=message).one_or_none() + assert created_motd + + # Get number of users + num_users = models.UnitUser.query.count() + + # Attempt request and catch email + with mail.record_messages() as outbox: + response: werkzeug.test.WrapperTestResponse = client.post( + tests.DDSEndpoint.MOTD_SEND, + headers=token, + json={"motd_id": created_motd.id, "unit_personnel_only": True}, ) assert response.status_code == http.HTTPStatus.OK assert len(outbox) == num_users From 9d30812b1bea45efea1b6b7557b008813a7e5f0b Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 12 Sep 2024 14:07:19 +0200 Subject: [PATCH 02/59] tests for new functionality --- tests/api/test_superadmin_only.py | 29 ++++++++++++++++++++++ tests/tests_v3/api/test_superadmin_only.py | 29 ++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/tests/api/test_superadmin_only.py b/tests/api/test_superadmin_only.py index 9bf439be0..c6856d4b3 100644 --- a/tests/api/test_superadmin_only.py +++ b/tests/api/test_superadmin_only.py @@ -588,6 +588,35 @@ def test_send_motd_no_primary_email(client: flask.testing.FlaskClient) -> None: assert "incorrect subject" not in outbox[-1].subject +def test_send_motd_incorrect_type_unit_personnel_only(client: flask.testing.FlaskClient) -> None: + """The parameter unit_personnel_only should be a boolean""" + # Authenticate + token: typing.Dict = get_token(username=users["Super Admin"], client=client) + + # Create a motd + message: str = "This is a message that should become a MOTD and then be sent to all the users." + new_motd: models.MOTD = models.MOTD(message=message) + db.session.add(new_motd) + db.session.commit() + + # Make sure the motd is created + created_motd: models.MOTD = models.MOTD.query.filter_by(message=message).one_or_none() + assert created_motd + + # Attempt request + with unittest.mock.patch.object(flask_mail.Connection, "send") as mock_mail_send: + response: werkzeug.test.WrapperTestResponse = client.post( + tests.DDSEndpoint.MOTD_SEND, + headers=token, + json={"motd_id": created_motd.id, "unit_personnel_only": "False"}, + ) + assert response.status_code == http.HTTPStatus.BAD_REQUEST + assert "The 'unit_personnel_only' argument must be a boolean." in response.json.get( + "message" + ) + assert mock_mail_send.call_count == 0 + + def test_send_motd_ok_all(client: flask.testing.FlaskClient) -> None: """Send a motd to all users.""" # Authenticate diff --git a/tests/tests_v3/api/test_superadmin_only.py b/tests/tests_v3/api/test_superadmin_only.py index 43af9e7df..4cb7d16ca 100644 --- a/tests/tests_v3/api/test_superadmin_only.py +++ b/tests/tests_v3/api/test_superadmin_only.py @@ -575,6 +575,35 @@ def test_send_motd_no_primary_email(client: flask.testing.FlaskClient) -> None: assert "incorrect subject" not in outbox[-1].subject +def test_send_motd_incorrect_type_unit_personnel_only(client: flask.testing.FlaskClient) -> None: + """The parameter unit_personnel_only should be a boolean""" + # Authenticate + token: typing.Dict = get_token(username=users["Super Admin"], client=client) + + # Create a motd + message: str = "This is a message that should become a MOTD and then be sent to all the users." + new_motd: models.MOTD = models.MOTD(message=message) + db.session.add(new_motd) + db.session.commit() + + # Make sure the motd is created + created_motd: models.MOTD = models.MOTD.query.filter_by(message=message).one_or_none() + assert created_motd + + # Attempt request + with unittest.mock.patch.object(flask_mail.Connection, "send") as mock_mail_send: + response: werkzeug.test.WrapperTestResponse = client.post( + tests.DDSEndpoint.MOTD_SEND, + headers=token, + json={"motd_id": created_motd.id, "unit_personnel_only": "False"}, + ) + assert response.status_code == http.HTTPStatus.BAD_REQUEST + assert "The 'unit_personnel_only' argument must be a boolean." in response.json.get( + "message" + ) + assert mock_mail_send.call_count == 0 + + def test_send_motd_ok_all(client: flask.testing.FlaskClient) -> None: """Send a motd to all users.""" # Authenticate From ce9e7c6404422beefb38b9209d00224d7d87602f Mon Sep 17 00:00:00 2001 From: valyo <582646+valyo@users.noreply.github.com> Date: Wed, 25 Sep 2024 10:18:08 +0200 Subject: [PATCH 03/59] bump docker/build-push-action to v5 --- .github/workflows/publish_and_trivyscan.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish_and_trivyscan.yml b/.github/workflows/publish_and_trivyscan.yml index 7bd44a6c8..ac1d90d0c 100644 --- a/.github/workflows/publish_and_trivyscan.yml +++ b/.github/workflows/publish_and_trivyscan.yml @@ -111,7 +111,7 @@ jobs: - name: Ensure lowercase name run: echo IMAGE_REPOSITORY=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV - name: Build for scan - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: file: Dockerfiles/backend.Dockerfile context: . @@ -130,7 +130,7 @@ jobs: sarif_file: "trivy-results.sarif" category: trivy-build - name: Publish image - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: file: Dockerfiles/backend.Dockerfile context: . From bf139a0865ad470d6dbec74aa97bf92d1da08238 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Wed, 25 Sep 2024 16:40:23 +0200 Subject: [PATCH 04/59] convert to float --- dds_web/commands.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dds_web/commands.py b/dds_web/commands.py index 7cd3d8777..03d703a45 100644 --- a/dds_web/commands.py +++ b/dds_web/commands.py @@ -85,7 +85,7 @@ def fill_db_wrapper(db_type): @click.option("--days_in_available", "-da", type=int, required=False, default=90) @click.option("--days_in_expired", "-de", type=int, required=False, default=30) @click.option("--quota", "-q", type=int, required=True) -@click.option("--warn-at", "-w", type=int, required=False, default=80) +@click.option("--warn-at", "-w", type=float, required=False, default=0.8) @flask.cli.with_appcontext def create_new_unit( name, @@ -1256,7 +1256,7 @@ def monitor_usage(): # Get info from database quota: int = unit.quota - warn_after: int = unit.warning_level + warn_after: float = unit.warning_level current_usage: int = unit.size # Check if 0 and then skip the next steps @@ -1273,7 +1273,7 @@ def monitor_usage(): # Information to log and potentially send info_string: str = ( f"- Quota:{quota} bytes\n" - f"- Warning level: {warn_after*quota} bytes ({warn_after*100}%)\n" + f"- Warning level: {int(warn_after*quota)} bytes ({int(warn_after*100)}%)\n" f"- Current usage: {current_usage} bytes ({perc_used}%)\n" ) flask.current_app.logger.debug( From d4c46fb0dae96832e546ecaaee290e93f9a83fc5 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 26 Sep 2024 10:08:04 +0200 Subject: [PATCH 05/59] added checks --- dds_web/commands.py | 2 +- dds_web/database/models.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/dds_web/commands.py b/dds_web/commands.py index 03d703a45..a290cd7cc 100644 --- a/dds_web/commands.py +++ b/dds_web/commands.py @@ -85,7 +85,7 @@ def fill_db_wrapper(db_type): @click.option("--days_in_available", "-da", type=int, required=False, default=90) @click.option("--days_in_expired", "-de", type=int, required=False, default=30) @click.option("--quota", "-q", type=int, required=True) -@click.option("--warn-at", "-w", type=float, required=False, default=0.8) +@click.option("--warn-at", "-w", type=click.FloatRange(0.0, 1.0), required=False, default=0.8) @flask.cli.with_appcontext def create_new_unit( name, diff --git a/dds_web/database/models.py b/dds_web/database/models.py index 104b413af..eb992339f 100644 --- a/dds_web/database/models.py +++ b/dds_web/database/models.py @@ -11,6 +11,7 @@ # Installed import sqlalchemy +from sqlalchemy.orm import validates import flask import argon2 import flask_login @@ -225,6 +226,12 @@ def size(self): return sum([p.size for p in self.projects]) + @validates("warning_level") + def validate_level(self, key, value): + if not (0.0 <= value <= 1.0): + raise ValueError("Warning level must be a float between 0 and 1") + return value + class Project(db.Model): """ From f7fb79421cbb4f656a8791c02336aae8ae1b3fc4 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 26 Sep 2024 10:11:50 +0200 Subject: [PATCH 06/59] sprintlog --- SPRINTLOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index c8c7c7d62..a72610a69 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -430,3 +430,4 @@ _Nothing merged during this sprint_ - Flask command to update unit quotas ([#1551](https://github.com/ScilifelabDataCentre/dds_web/pull/1551)) - Bump python base image to 3.12 and related libraries in both web and client([#1548](https://github.com/ScilifelabDataCentre/dds_web/pull/1548)) +- Warning_level option defaults to 0.8([#1557](https://github.com/ScilifelabDataCentre/dds_web/pull/1557)) From 173e5d761aa20f1028a32aad85d0f0883b255c7e Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 26 Sep 2024 13:45:51 +0200 Subject: [PATCH 07/59] test for commands --- tests/test_commands.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_commands.py b/tests/test_commands.py index cfb4de2b0..1450c7c8a 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -134,6 +134,25 @@ def test_create_new_unit_public_id_too_long(client, runner, capfd: LogCaptureFix ) +def test_create_new_unit_incorrect_warning_level(client, runner, capfd: LogCaptureFixture) -> None: + """Create new unit, warning level is not a float between 0.0 and 1.0""" + # Change public_id + incorrect_unit: typing.Dict = correct_unit.copy() + incorrect_unit["warn-at"] = 30 + + # Get command options + command_options = create_command_options_from_dict(options=incorrect_unit) + + # Run command + result: click.testing.Result = runner.invoke(create_new_unit, command_options) + + assert result.exit_code != 0 # No sucess + # Verify that unit doesn't exist + assert ( + not db.session.query(models.Unit).filter(models.Unit.name == incorrect_unit["name"]).all() + ) + + def test_create_new_unit_public_id_incorrect_characters( client, runner, capfd: LogCaptureFixture ) -> None: From 4714c1b554b7d428ad7f722e62c7d40932e15e0b Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 26 Sep 2024 18:39:32 +0200 Subject: [PATCH 08/59] test for models --- tests/test_models.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/test_models.py b/tests/test_models.py index 6dc559dc3..316611b24 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -109,6 +109,32 @@ def test_delete_unit_row(client): assert invites == [] +def test_create_unit_wrong_warning_level(client): + """Test try to create a unit which has an invalid value for the warning level""" + from dds_web.utils import current_time + + unit = models.Unit.query.filter_by(name="Unit 1").first() + + with pytest.raises(ValueError) as err: + new_unit = models.Unit( + name="test", + public_id="public_id", + external_display_name="external_display_name", + contact_email=unit.contact_email, + internal_ref="public_id", + sto4_start_time=current_time(), + sto4_endpoint=unit.sto4_endpoint, + sto4_name=unit.sto4_name, + sto4_access=unit.sto4_access, + sto4_secret=unit.sto4_secret, + days_in_available=unit.days_in_available, + days_in_expired=unit.days_in_expired, + quota=unit.quota, + warning_level=20.0, + ) + assert "Warning level must be a float between 0 and 1" in str(err.value) + + # Project #################################################################################### Project # def __setup_project(): """ From 48df40382f0bfc4b141f897990cff50e2d1c369e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Revuelta?= <46089290+rv0lt@users.noreply.github.com> Date: Thu, 3 Oct 2024 09:35:06 +0200 Subject: [PATCH 09/59] Update tests/api/test_superadmin_only.py Co-authored-by: Valentin Georgiev --- tests/api/test_superadmin_only.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/api/test_superadmin_only.py b/tests/api/test_superadmin_only.py index c6856d4b3..51491f963 100644 --- a/tests/api/test_superadmin_only.py +++ b/tests/api/test_superadmin_only.py @@ -608,7 +608,7 @@ def test_send_motd_incorrect_type_unit_personnel_only(client: flask.testing.Flas response: werkzeug.test.WrapperTestResponse = client.post( tests.DDSEndpoint.MOTD_SEND, headers=token, - json={"motd_id": created_motd.id, "unit_personnel_only": "False"}, + json={"motd_id": created_motd.id, "unit_personnel_only": "some_string"}, ) assert response.status_code == http.HTTPStatus.BAD_REQUEST assert "The 'unit_personnel_only' argument must be a boolean." in response.json.get( From 383f39cbf1119f07870147a1c86a4db8e9549bef Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 3 Oct 2024 09:42:02 +0200 Subject: [PATCH 10/59] Sprintlog --- SPRINTLOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index a72610a69..1d26bd55d 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -431,3 +431,4 @@ _Nothing merged during this sprint_ - Flask command to update unit quotas ([#1551](https://github.com/ScilifelabDataCentre/dds_web/pull/1551)) - Bump python base image to 3.12 and related libraries in both web and client([#1548](https://github.com/ScilifelabDataCentre/dds_web/pull/1548)) - Warning_level option defaults to 0.8([#1557](https://github.com/ScilifelabDataCentre/dds_web/pull/1557)) +- Modify the motd command to only send to unit users([#1552](https://github.com/ScilifelabDataCentre/dds_web/pull/1552)) From e91348363bd0c6c787cd126e9069e7b260106e32 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 3 Oct 2024 10:18:55 +0200 Subject: [PATCH 11/59] better explained string --- tests/tests_v3/api/test_superadmin_only.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_v3/api/test_superadmin_only.py b/tests/tests_v3/api/test_superadmin_only.py index 4cb7d16ca..a02327f9f 100644 --- a/tests/tests_v3/api/test_superadmin_only.py +++ b/tests/tests_v3/api/test_superadmin_only.py @@ -595,7 +595,7 @@ def test_send_motd_incorrect_type_unit_personnel_only(client: flask.testing.Flas response: werkzeug.test.WrapperTestResponse = client.post( tests.DDSEndpoint.MOTD_SEND, headers=token, - json={"motd_id": created_motd.id, "unit_personnel_only": "False"}, + json={"motd_id": created_motd.id, "unit_personnel_only": "some_string"}, ) assert response.status_code == http.HTTPStatus.BAD_REQUEST assert "The 'unit_personnel_only' argument must be a boolean." in response.json.get( From 775754024bea79622021c5b79b053806dd8f3459 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 3 Oct 2024 11:16:36 +0200 Subject: [PATCH 12/59] clarify in returned message if sent to only personnel only --- dds_web/api/superadmin_only.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dds_web/api/superadmin_only.py b/dds_web/api/superadmin_only.py index 42be6e724..a72986c50 100644 --- a/dds_web/api/superadmin_only.py +++ b/dds_web/api/superadmin_only.py @@ -210,8 +210,14 @@ def post(self): ) # Send email utils.send_email_with_retry(msg=msg, obj=conn) + + return_msg = f"MOTD '{motd_id}' has been " + if unit_personnel_only: + return_msg += "sent to unit personnel only." + else: + return_msg += "sent to all users." - return {"message": f"MOTD '{motd_id}' has been sent to the users."} + return {"message": return_msg} class FindUser(flask_restful.Resource): From f94840eed98e4eb4be1cf2007da9e7b8cf20980b Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 3 Oct 2024 11:18:24 +0200 Subject: [PATCH 13/59] black --- dds_web/api/superadmin_only.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/api/superadmin_only.py b/dds_web/api/superadmin_only.py index a72986c50..bb7de7c0b 100644 --- a/dds_web/api/superadmin_only.py +++ b/dds_web/api/superadmin_only.py @@ -210,7 +210,7 @@ def post(self): ) # Send email utils.send_email_with_retry(msg=msg, obj=conn) - + return_msg = f"MOTD '{motd_id}' has been " if unit_personnel_only: return_msg += "sent to unit personnel only." From d61b0c9c800fc6d0053ac17f13537f046cb5783d Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 3 Oct 2024 11:29:49 +0200 Subject: [PATCH 14/59] sprintlog --- SPRINTLOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index 1d26bd55d..53172e234 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -431,4 +431,7 @@ _Nothing merged during this sprint_ - Flask command to update unit quotas ([#1551](https://github.com/ScilifelabDataCentre/dds_web/pull/1551)) - Bump python base image to 3.12 and related libraries in both web and client([#1548](https://github.com/ScilifelabDataCentre/dds_web/pull/1548)) - Warning_level option defaults to 0.8([#1557](https://github.com/ScilifelabDataCentre/dds_web/pull/1557)) + +# 2024-09-24 - 2024-10-04 + - Modify the motd command to only send to unit users([#1552](https://github.com/ScilifelabDataCentre/dds_web/pull/1552)) From cc698e7fbf4bf285b6a1a20945fc2249634b48f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Revuelta?= <46089290+rv0lt@users.noreply.github.com> Date: Thu, 3 Oct 2024 15:12:54 +0200 Subject: [PATCH 15/59] Update SPRINTLOG.md Co-authored-by: Valentin Georgiev --- SPRINTLOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index 53172e234..079acfd7c 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -434,4 +434,4 @@ _Nothing merged during this sprint_ # 2024-09-24 - 2024-10-04 -- Modify the motd command to only send to unit users([#1552](https://github.com/ScilifelabDataCentre/dds_web/pull/1552)) +- Add option to motd command for sending to unit users only([#1552](https://github.com/ScilifelabDataCentre/dds_web/pull/1552)) From 6aec4782b76485feb7ae1f7a2045d6acf1cd0743 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 3 Oct 2024 15:20:22 +0200 Subject: [PATCH 16/59] sprintlog --- SPRINTLOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index a72610a69..48344a5ba 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -431,3 +431,7 @@ _Nothing merged during this sprint_ - Flask command to update unit quotas ([#1551](https://github.com/ScilifelabDataCentre/dds_web/pull/1551)) - Bump python base image to 3.12 and related libraries in both web and client([#1548](https://github.com/ScilifelabDataCentre/dds_web/pull/1548)) - Warning_level option defaults to 0.8([#1557](https://github.com/ScilifelabDataCentre/dds_web/pull/1557)) + +# 2024-09-24 - 2024-10-04 + +- Update readme to remove mention of DockerHub registry([#1558](https://github.com/ScilifelabDataCentre/dds_web/pull/1558)) From bfcaaf5c8d2474ba9b427a4b706c55f7db886842 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 3 Oct 2024 15:20:42 +0200 Subject: [PATCH 17/59] remove dockerhub mention --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0bb2111a0..d060422be 100644 --- a/README.md +++ b/README.md @@ -257,7 +257,7 @@ Equally, if you want to tear down you need to run pytest _twice_ without it, as ## Production Instance -The production version of the backend image is published at [Dockerhub](https://hub.docker.com/repository/docker/scilifelabdatacentre/dds-backend). It can also be built by running: +The production version of the backend image is published at the [GitHub Container registry](ghcr.io/scilifelabdatacentre/dds-backend). It can also be built by running: ```bash docker build --target production -f Dockerfiles/backend.Dockerfile . From 68554e3c7a830f5677ee059e235f93e49eb4800e Mon Sep 17 00:00:00 2001 From: rv0lt Date: Mon, 7 Oct 2024 12:15:45 +0200 Subject: [PATCH 18/59] fix actions to download artifacts --- .github/workflows/publish_and_trivyscan.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish_and_trivyscan.yml b/.github/workflows/publish_and_trivyscan.yml index ac1d90d0c..ab3f56bea 100644 --- a/.github/workflows/publish_and_trivyscan.yml +++ b/.github/workflows/publish_and_trivyscan.yml @@ -96,12 +96,12 @@ jobs: uses: actions/download-artifact@v4 with: name: technical-overview-pdf - path: dds_web/static/dds-technical-overview.pdf + path: dds_web/static - name: Download troubleshooting PDF uses: actions/download-artifact@v4 with: name: troubleshooting-pdf - path: dds_web/static/dds-troubleshooting.pdf + path: dds_web/static - name: Docker metadata id: meta uses: docker/metadata-action@v5 From 750460542d51b11ab6c0ef3f3af24c12af2aded5 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Mon, 7 Oct 2024 12:17:59 +0200 Subject: [PATCH 19/59] sprintlog --- SPRINTLOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index 079acfd7c..e295d3e07 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -435,3 +435,7 @@ _Nothing merged during this sprint_ # 2024-09-24 - 2024-10-04 - Add option to motd command for sending to unit users only([#1552](https://github.com/ScilifelabDataCentre/dds_web/pull/1552)) + +# 2024-10-07 - 2024-10-18 + +- Fix action to download the pdfs to correct directory([#1559](https://github.com/ScilifelabDataCentre/dds_web/pull/1559)) From a5a4b594fcae9c4932d5dcb46932e65c432dba7a Mon Sep 17 00:00:00 2001 From: rv0lt Date: Mon, 7 Oct 2024 12:19:35 +0200 Subject: [PATCH 20/59] move to current sprint --- SPRINTLOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index 9ebca4b99..784da222a 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -434,5 +434,8 @@ _Nothing merged during this sprint_ # 2024-09-24 - 2024-10-04 -- Update readme to remove mention of DockerHub registry([#1558](https://github.com/ScilifelabDataCentre/dds_web/pull/1558)) - Add option to motd command for sending to unit users only([#1552](https://github.com/ScilifelabDataCentre/dds_web/pull/1552)) + +# 2024-10-07 - 2024-10-18 + +- Update readme to remove mention of DockerHub registry([#1558](https://github.com/ScilifelabDataCentre/dds_web/pull/1558)) From 7e07a16a77eeaa318a5bf5c475553b6ab0774819 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Revuelta?= <46089290+rv0lt@users.noreply.github.com> Date: Tue, 8 Oct 2024 11:31:33 +0200 Subject: [PATCH 21/59] Update SPRINTLOG.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ina Odén Österbo <35953392+i-oden@users.noreply.github.com> --- SPRINTLOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index e295d3e07..808ed5721 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -438,4 +438,4 @@ _Nothing merged during this sprint_ # 2024-10-07 - 2024-10-18 -- Fix action to download the pdfs to correct directory([#1559](https://github.com/ScilifelabDataCentre/dds_web/pull/1559)) +- Workflow bug fixed: PDFs (Technical Overview and Troubleshooting) were downloaded to incorrect directory([#1559](https://github.com/ScilifelabDataCentre/dds_web/pull/1559)) From bbb1f3be2fc67ba0f4799aa016c58497399f2a89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Revuelta?= <46089290+rv0lt@users.noreply.github.com> Date: Tue, 8 Oct 2024 13:03:45 +0200 Subject: [PATCH 22/59] Update SPRINTLOG.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ina Odén Österbo <35953392+i-oden@users.noreply.github.com> --- SPRINTLOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index 784da222a..3101c19fd 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -438,4 +438,4 @@ _Nothing merged during this sprint_ # 2024-10-07 - 2024-10-18 -- Update readme to remove mention of DockerHub registry([#1558](https://github.com/ScilifelabDataCentre/dds_web/pull/1558)) +- Update readme: backend image is published to GHCR, not DockerHub ([#1558](https://github.com/ScilifelabDataCentre/dds_web/pull/1558)) From 6ae6bf2c408b8f646e84449d8bd62b2302566bdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Revuelta?= <46089290+rv0lt@users.noreply.github.com> Date: Tue, 8 Oct 2024 13:04:17 +0200 Subject: [PATCH 23/59] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ina Odén Österbo <35953392+i-oden@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d060422be..f5be819d3 100644 --- a/README.md +++ b/README.md @@ -257,7 +257,7 @@ Equally, if you want to tear down you need to run pytest _twice_ without it, as ## Production Instance -The production version of the backend image is published at the [GitHub Container registry](ghcr.io/scilifelabdatacentre/dds-backend). It can also be built by running: +The production version of the backend image is published at the [GitHub Container Registry (GHCR)](ghcr.io/scilifelabdatacentre/dds-backend). It can also be built by running: ```bash docker build --target production -f Dockerfiles/backend.Dockerfile . From dcf4c3a666c8f3722240ee2145de2b24714a52dd Mon Sep 17 00:00:00 2001 From: rv0lt Date: Tue, 8 Oct 2024 13:04:57 +0200 Subject: [PATCH 24/59] prettier --- SPRINTLOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index 3101c19fd..532267fce 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -438,4 +438,4 @@ _Nothing merged during this sprint_ # 2024-10-07 - 2024-10-18 -- Update readme: backend image is published to GHCR, not DockerHub ([#1558](https://github.com/ScilifelabDataCentre/dds_web/pull/1558)) +- Update readme: backend image is published to GHCR, not DockerHub ([#1558](https://github.com/ScilifelabDataCentre/dds_web/pull/1558)) From 7a807cb966925cba4a5406843e6a03b620d00d64 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Tue, 8 Oct 2024 13:49:04 +0200 Subject: [PATCH 25/59] ! Added changes to show instance name --- dds_web/commands.py | 5 +++++ dds_web/config.py | 1 + 2 files changed, 6 insertions(+) diff --git a/dds_web/commands.py b/dds_web/commands.py index a290cd7cc..a8c69aff3 100644 --- a/dds_web/commands.py +++ b/dds_web/commands.py @@ -841,10 +841,15 @@ def monthly_usage(): send_email_with_retry, ) + instance_name = flask.current_app.config.get("INSTANCE_NAME") # Email settings email_recipient: str = flask.current_app.config.get("MAIL_DDS") # -- Success email_subject: str = "[INVOICING CRONJOB]" + # instance name can be none, so check if it is set and add it to the subject + if instance_name: + email_subject += f" ({instance_name})" + email_body: str = ( "The calculation of the monthly usage succeeded; The byte hours " "for all active projects have been saved to the database." diff --git a/dds_web/config.py b/dds_web/config.py index be2aa4254..1085311d1 100644 --- a/dds_web/config.py +++ b/dds_web/config.py @@ -18,6 +18,7 @@ class Config(object): """Base config""" SITE_NAME = "Data Delivery System" + INSTANCE_NAME = "DEVELOPMENT" SECRET_KEY = "REPLACE_THE_STRING_IN_PRODUCTION" # DB related config From 651b5b18398711d883af6a11f81bf18f9b3950a1 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Tue, 8 Oct 2024 13:50:04 +0200 Subject: [PATCH 26/59] added server name --- dds_web/commands.py | 2 +- dds_web/config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dds_web/commands.py b/dds_web/commands.py index a8c69aff3..94d3fa0d4 100644 --- a/dds_web/commands.py +++ b/dds_web/commands.py @@ -847,7 +847,7 @@ def monthly_usage(): # -- Success email_subject: str = "[INVOICING CRONJOB]" # instance name can be none, so check if it is set and add it to the subject - if instance_name: + if server_name: email_subject += f" ({instance_name})" email_body: str = ( diff --git a/dds_web/config.py b/dds_web/config.py index 1085311d1..e4389e3dd 100644 --- a/dds_web/config.py +++ b/dds_web/config.py @@ -18,7 +18,7 @@ class Config(object): """Base config""" SITE_NAME = "Data Delivery System" - INSTANCE_NAME = "DEVELOPMENT" + SERVER_NAME = "DEVELOPMENT" SECRET_KEY = "REPLACE_THE_STRING_IN_PRODUCTION" # DB related config From 83a2df41534b5ab87dc5f76fe19037c2c2108c13 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Wed, 9 Oct 2024 10:21:32 +0200 Subject: [PATCH 27/59] prettier --- SPRINTLOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index 7593aff8b..c955ccfc6 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -440,4 +440,3 @@ _Nothing merged during this sprint_ - Update readme: backend image is published to GHCR, not DockerHub ([#1558](https://github.com/ScilifelabDataCentre/dds_web/pull/1558)) - Workflow bug fixed: PDFs (Technical Overview and Troubleshooting) were downloaded to incorrect directory([#1559](https://github.com/ScilifelabDataCentre/dds_web/pull/1559)) - From ce5eac4d6018b9b47c1dba5a7f61d0c55c02c1c8 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Wed, 9 Oct 2024 10:44:50 +0200 Subject: [PATCH 28/59] update trivy action --- .github/workflows/publish_and_trivyscan.yml | 4 +++- .github/workflows/trivy-scheduled-dev.yml | 4 +++- .github/workflows/trivy-scheduled-master.yml | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish_and_trivyscan.yml b/.github/workflows/publish_and_trivyscan.yml index ab3f56bea..f250b7fba 100644 --- a/.github/workflows/publish_and_trivyscan.yml +++ b/.github/workflows/publish_and_trivyscan.yml @@ -118,7 +118,9 @@ jobs: push: false tags: ghcr.io/${{ env.IMAGE_REPOSITORY }}:sha-${{ github.sha }} - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.7.1 + uses: aquasecurity/trivy-action@0.26.0 + env: + TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db,public.ecr.aws/aquasecurity/trivy-db with: image-ref: "ghcr.io/${{ env.IMAGE_REPOSITORY }}:sha-${{ github.sha }}" format: "sarif" diff --git a/.github/workflows/trivy-scheduled-dev.yml b/.github/workflows/trivy-scheduled-dev.yml index 1399be061..7bd34cf9d 100644 --- a/.github/workflows/trivy-scheduled-dev.yml +++ b/.github/workflows/trivy-scheduled-dev.yml @@ -23,7 +23,9 @@ jobs: run: echo REPOSITORY_OWNER=$(echo ${{ github.repository_owner }} | tr "[:upper:]" "[:lower:]") >> $GITHUB_ENV - name: Run Trivy on latest dev image - uses: aquasecurity/trivy-action@0.24.0 + uses: aquasecurity/trivy-action@0.26.0 + env: + TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db,public.ecr.aws/aquasecurity/trivy-db with: image-ref: "ghcr.io/${{ env.REPOSITORY_OWNER }}/dds-backend:dev" format: "sarif" diff --git a/.github/workflows/trivy-scheduled-master.yml b/.github/workflows/trivy-scheduled-master.yml index 4ef9fa58b..e2ef46b86 100644 --- a/.github/workflows/trivy-scheduled-master.yml +++ b/.github/workflows/trivy-scheduled-master.yml @@ -25,7 +25,9 @@ jobs: run: echo REPOSITORY_OWNER=$(echo ${{ github.repository_owner }} | tr "[:upper:]" "[:lower:]") >> $GITHUB_ENV - name: Run Trivy on latest release image - uses: aquasecurity/trivy-action@0.24.0 + uses: aquasecurity/trivy-action@0.26.0 + env: + TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db,public.ecr.aws/aquasecurity/trivy-db with: image-ref: "ghcr.io/${{ env.REPOSITORY_OWNER }}/dds-backend:latest" format: "sarif" From dcbff9146b83698164b806cafa4c8db124574bc5 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Wed, 9 Oct 2024 10:57:31 +0200 Subject: [PATCH 29/59] new instance name variable --- dds_web/commands.py | 2 +- dds_web/config.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dds_web/commands.py b/dds_web/commands.py index 94d3fa0d4..a8c69aff3 100644 --- a/dds_web/commands.py +++ b/dds_web/commands.py @@ -847,7 +847,7 @@ def monthly_usage(): # -- Success email_subject: str = "[INVOICING CRONJOB]" # instance name can be none, so check if it is set and add it to the subject - if server_name: + if instance_name: email_subject += f" ({instance_name})" email_body: str = ( diff --git a/dds_web/config.py b/dds_web/config.py index e4389e3dd..1c50726fb 100644 --- a/dds_web/config.py +++ b/dds_web/config.py @@ -17,8 +17,8 @@ class Config(object): """Base config""" - SITE_NAME = "Data Delivery System" - SERVER_NAME = "DEVELOPMENT" + SITE_NAME = "Data Delivery System" # Name of the site, will appear in the website title + INSTANCE_NAME = "DEVELOPMENT" # Name of the instance, e.g. "PRODUCTION", "DEVELOPMENT", "TEST" SECRET_KEY = "REPLACE_THE_STRING_IN_PRODUCTION" # DB related config From a7342b427624f3f1b338fefd26c91b65c495e998 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Wed, 9 Oct 2024 12:36:24 +0200 Subject: [PATCH 30/59] Add option to send usage as well --- dds_web/commands.py | 17 +++++++++++++++-- dds_web/config.py | 4 ++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/dds_web/commands.py b/dds_web/commands.py index a8c69aff3..742292209 100644 --- a/dds_web/commands.py +++ b/dds_web/commands.py @@ -841,13 +841,14 @@ def monthly_usage(): send_email_with_retry, ) + # Get the instance name (DEVELOPMENT, PRODUCTION, etc.) instance_name = flask.current_app.config.get("INSTANCE_NAME") + # Email settings email_recipient: str = flask.current_app.config.get("MAIL_DDS") # -- Success email_subject: str = "[INVOICING CRONJOB]" - # instance name can be none, so check if it is set and add it to the subject - if instance_name: + if instance_name: # instance name can be none, so check if it is set and add it to the subject email_subject += f" ({instance_name})" email_body: str = ( @@ -856,6 +857,9 @@ def monthly_usage(): ) # -- Failure error_subject: str = f"{email_subject} Error in monthly-usage cronjob" + if instance_name: # instance name can be none, so check if it is set and add it to the subject + error_subject += f" ({instance_name})" + error_body: str = ( "There was an error in the cronjob 'monthly-usage', used for calculating the" " byte hours for every active project in the last month.\n\n" @@ -977,13 +981,22 @@ def send_usage(months): from dds_web.database import models from dds_web.utils import current_time, page_query, send_email_with_retry + # Get the instance name (DEVELOPMENT, PRODUCTION, etc.) + instance_name = flask.current_app.config.get("INSTANCE_NAME") + # Email settings email_recipient: str = flask.current_app.config.get("MAIL_DDS") # -- Success email_subject: str = "[SEND-USAGE CRONJOB]" + if instance_name: # instance name can be none, so check if it is set and add it to the subject + email_subject += f" ({instance_name})" + email_body: str = f"Here is the usage for the last {months} months.\n" # -- Failure error_subject: str = f"{email_subject} Error in send-usage cronjob" + if instance_name: # instance name can be none, so check if it is set and add it to the subject + error_subject += f" ({instance_name})" + error_body: str = ( "There was an error in the cronjob 'send-usage', used for sending" " information about the storage usage for each SciLifeLab unit. \n\n" diff --git a/dds_web/config.py b/dds_web/config.py index 1c50726fb..f40e8bdd6 100644 --- a/dds_web/config.py +++ b/dds_web/config.py @@ -17,8 +17,8 @@ class Config(object): """Base config""" - SITE_NAME = "Data Delivery System" # Name of the site, will appear in the website title - INSTANCE_NAME = "DEVELOPMENT" # Name of the instance, e.g. "PRODUCTION", "DEVELOPMENT", "TEST" + SITE_NAME = "Data Delivery System" # Name of the site, will appear in the website title + INSTANCE_NAME = "DEVELOPMENT" # Name of the instance, e.g. "PRODUCTION", "DEVELOPMENT", "TEST" SECRET_KEY = "REPLACE_THE_STRING_IN_PRODUCTION" # DB related config From 9262b464fb54e50f6e003757b318a9d1f44b5e68 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Wed, 9 Oct 2024 12:38:22 +0200 Subject: [PATCH 31/59] sprintlog --- SPRINTLOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index c955ccfc6..b4ce8a07a 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -440,3 +440,4 @@ _Nothing merged during this sprint_ - Update readme: backend image is published to GHCR, not DockerHub ([#1558](https://github.com/ScilifelabDataCentre/dds_web/pull/1558)) - Workflow bug fixed: PDFs (Technical Overview and Troubleshooting) were downloaded to incorrect directory([#1559](https://github.com/ScilifelabDataCentre/dds_web/pull/1559)) +- Modify the invoicing commands to send the instance name in the emails([#1561](https://github.com/ScilifelabDataCentre/dds_web/pull/1561)) From 05e94efef16298722d9f54e9d76b07c9f9ea6313 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Wed, 9 Oct 2024 12:42:32 +0200 Subject: [PATCH 32/59] add instance name to stats command --- dds_web/commands.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dds_web/commands.py b/dds_web/commands.py index 742292209..c4550956f 100644 --- a/dds_web/commands.py +++ b/dds_web/commands.py @@ -1149,9 +1149,15 @@ def collect_stats(): # Get current time current_time = dds_web.utils.timestamp(ts_format="%Y-%m-%d") + # Get the instance name (DEVELOPMENT, PRODUCTION, etc.) + instance_name = flask.current_app.config.get("INSTANCE_NAME") + # Get email address recipient: str = flask.current_app.config.get("MAIL_DDS") error_subject: str = "[CRONJOB] Error during collection of DDS unit- and user statistics." + if instance_name: # instance name can be none, so check if it is set and add it to the subject + error_subject += f" ({instance_name})" + error_body: str = ( f"The cronjob 'reporting' experienced issues. Please see logs. Time: {current_time}." ) From 5e6f8f758796a943384fc6fa1c504fc22f05c866 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Wed, 9 Oct 2024 14:04:35 +0200 Subject: [PATCH 33/59] update test for new email subject --- tests/test_commands.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 1450c7c8a..3370569f2 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1655,7 +1655,10 @@ def create_file_versions(project: models.Project): # Error email should be sent assert len(outbox1) == 1 - assert "[INVOICING CRONJOB] Error in monthly-usage cronjob" in outbox1[-1].subject + assert ( + "[INVOICING CRONJOB] (DEVELOPMENT) Error in monthly-usage cronjob" + in outbox1[-1].subject + ) assert "What to do:" in outbox1[-1].body # No usage rows should have been saved @@ -1688,7 +1691,10 @@ def create_file_versions(project: models.Project): # Error email should have been sent assert len(outbox2) == 1 - assert "[INVOICING CRONJOB] Error in monthly-usage cronjob" in outbox2[-1].subject + assert ( + "[INVOICING CRONJOB] (DEVELOPMENT) Error in monthly-usage cronjob" + in outbox2[-1].subject + ) assert "What to do:" in outbox2[-1].body # Project versions should not be altered @@ -1721,7 +1727,10 @@ def create_file_versions(project: models.Project): # Email should be sent assert len(outbox3) == 1 - assert "[INVOICING CRONJOB] Usage records available for collection" in outbox3[-1].subject + assert ( + "[INVOICING CRONJOB] (DEVELOPMENT) Usage records available for collection" + in outbox3[-1].subject + ) assert ( "The calculation of the monthly usage succeeded; The byte hours for all active projects have been saved to the database." in outbox3[-1].body @@ -1918,7 +1927,7 @@ def run_command_and_check_output(months_to_test, start_time): # Verify output and sent email assert len(outbox) == 1 assert ( - "[SEND-USAGE CRONJOB] Usage records attached in the present mail" + "[SEND-USAGE CRONJOB] (DEVELOPMENT) Usage records attached in the present mail" in outbox[-1].subject ) assert f"Here is the usage for the last {months_to_test} months." in outbox[-1].body @@ -2055,5 +2064,8 @@ def test_send_usage_error_csv(client, cli_runner, capfd: LogCaptureFixture): # Verify error email :- At least one email was sent assert len(outbox) == 1 - assert "[SEND-USAGE CRONJOB] Error in send-usage cronjob" in outbox[-1].subject + assert ( + "[SEND-USAGE CRONJOB] (DEVELOPMENT) Error in send-usage cronjob" + in outbox[-1].subject + ) assert "There was an error in the cronjob 'send-usage'" in outbox[-1].body From 738592a38c9b30dcb97deeca820558cb93eb2985 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 10 Oct 2024 10:30:27 +0200 Subject: [PATCH 34/59] test for no instance name --- tests/test_commands.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 3370569f2..74809fdb7 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1657,7 +1657,7 @@ def create_file_versions(project: models.Project): assert len(outbox1) == 1 assert ( "[INVOICING CRONJOB] (DEVELOPMENT) Error in monthly-usage cronjob" - in outbox1[-1].subject + == outbox1[-1].subject ) assert "What to do:" in outbox1[-1].body @@ -1693,7 +1693,7 @@ def create_file_versions(project: models.Project): assert len(outbox2) == 1 assert ( "[INVOICING CRONJOB] (DEVELOPMENT) Error in monthly-usage cronjob" - in outbox2[-1].subject + == outbox2[-1].subject ) assert "What to do:" in outbox2[-1].body @@ -1747,7 +1747,25 @@ def create_file_versions(project: models.Project): assert usage_row_2 -# reporting units and users +def test_monthly_usage_no_instance_name(client, cli_runner, capfd: LogCaptureFixture): + """Test that the command do not send an email with the name if it is not set.""" + + import flask + + assert flask.current_app.config.get("INSTANCE_NAME") == "DEVELOPMENT" + # Set the instance name to none + flask.current_app.config["INSTANCE_NAME"] = None + + with mail.record_messages() as outbox: + cli_runner.invoke(monthly_usage) + + # Email should be sent + assert len(outbox) == 1 + assert "[INVOICING CRONJOB] Usage records available for collection" == outbox[-1].subject + assert ( + "The calculation of the monthly usage succeeded; The byte hours for all active projects have been saved to the database." + == outbox[-1].body + ) def test_collect_stats(client, cli_runner, fs: FakeFilesystem): @@ -2066,6 +2084,6 @@ def test_send_usage_error_csv(client, cli_runner, capfd: LogCaptureFixture): assert len(outbox) == 1 assert ( "[SEND-USAGE CRONJOB] (DEVELOPMENT) Error in send-usage cronjob" - in outbox[-1].subject + == outbox[-1].subject ) assert "There was an error in the cronjob 'send-usage'" in outbox[-1].body From 874dd25d7ac6f8c374b305acc585ffc377319f21 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 10 Oct 2024 10:30:35 +0200 Subject: [PATCH 35/59] feedback --- dds_web/commands.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dds_web/commands.py b/dds_web/commands.py index c4550956f..94d35a079 100644 --- a/dds_web/commands.py +++ b/dds_web/commands.py @@ -857,8 +857,6 @@ def monthly_usage(): ) # -- Failure error_subject: str = f"{email_subject} Error in monthly-usage cronjob" - if instance_name: # instance name can be none, so check if it is set and add it to the subject - error_subject += f" ({instance_name})" error_body: str = ( "There was an error in the cronjob 'monthly-usage', used for calculating the" @@ -994,8 +992,6 @@ def send_usage(months): email_body: str = f"Here is the usage for the last {months} months.\n" # -- Failure error_subject: str = f"{email_subject} Error in send-usage cronjob" - if instance_name: # instance name can be none, so check if it is set and add it to the subject - error_subject += f" ({instance_name})" error_body: str = ( "There was an error in the cronjob 'send-usage', used for sending" @@ -1155,8 +1151,6 @@ def collect_stats(): # Get email address recipient: str = flask.current_app.config.get("MAIL_DDS") error_subject: str = "[CRONJOB] Error during collection of DDS unit- and user statistics." - if instance_name: # instance name can be none, so check if it is set and add it to the subject - error_subject += f" ({instance_name})" error_body: str = ( f"The cronjob 'reporting' experienced issues. Please see logs. Time: {current_time}." From afa5d298a23e89af809b1e02eeeaa47e5e95f12c Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 10 Oct 2024 10:32:58 +0200 Subject: [PATCH 36/59] change in to == --- tests/test_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 74809fdb7..24e548b7d 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1729,7 +1729,7 @@ def create_file_versions(project: models.Project): assert len(outbox3) == 1 assert ( "[INVOICING CRONJOB] (DEVELOPMENT) Usage records available for collection" - in outbox3[-1].subject + == outbox3[-1].subject ) assert ( "The calculation of the monthly usage succeeded; The byte hours for all active projects have been saved to the database." From 0d2ac37a45c8d6038054e18ec9a878b2e8b455a0 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Fri, 11 Oct 2024 09:53:23 +0200 Subject: [PATCH 37/59] sprintlog --- SPRINTLOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index c955ccfc6..ac8acda5a 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -440,3 +440,4 @@ _Nothing merged during this sprint_ - Update readme: backend image is published to GHCR, not DockerHub ([#1558](https://github.com/ScilifelabDataCentre/dds_web/pull/1558)) - Workflow bug fixed: PDFs (Technical Overview and Troubleshooting) were downloaded to incorrect directory([#1559](https://github.com/ScilifelabDataCentre/dds_web/pull/1559)) +- Update trivy action and add a second mirror repository to reduce TOO MANY REQUEST issue([#1560](https://github.com/ScilifelabDataCentre/dds_web/pull/1560)) From daae0f547c5fa5c5435eddce6e1e88a518d76fd6 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Fri, 11 Oct 2024 11:32:10 +0200 Subject: [PATCH 38/59] remove ununsed variable --- dds_web/commands.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/dds_web/commands.py b/dds_web/commands.py index 94d35a079..e742bdc47 100644 --- a/dds_web/commands.py +++ b/dds_web/commands.py @@ -1145,9 +1145,6 @@ def collect_stats(): # Get current time current_time = dds_web.utils.timestamp(ts_format="%Y-%m-%d") - # Get the instance name (DEVELOPMENT, PRODUCTION, etc.) - instance_name = flask.current_app.config.get("INSTANCE_NAME") - # Get email address recipient: str = flask.current_app.config.get("MAIL_DDS") error_subject: str = "[CRONJOB] Error during collection of DDS unit- and user statistics." From 9877d31f464ce62d87c44d5dc4b58b97680f1ca2 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Fri, 11 Oct 2024 12:40:36 +0200 Subject: [PATCH 39/59] added instance name to stats email --- dds_web/commands.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dds_web/commands.py b/dds_web/commands.py index e742bdc47..6550a8679 100644 --- a/dds_web/commands.py +++ b/dds_web/commands.py @@ -1147,7 +1147,14 @@ def collect_stats(): # Get email address recipient: str = flask.current_app.config.get("MAIL_DDS") - error_subject: str = "[CRONJOB] Error during collection of DDS unit- and user statistics." + + # Get the instance name (DEVELOPMENT, PRODUCTION, etc.) + instance_name = flask.current_app.config.get("INSTANCE_NAME") + + error_subject: str = "[CRONJOB]" + if instance_name: # instance name can be none, so check if it is set and add it to the subject + error_subject += f" ({instance_name})" + error_subject += " Error during collection of DDS unit and user statistics." error_body: str = ( f"The cronjob 'reporting' experienced issues. Please see logs. Time: {current_time}." From ded17bdf6a79bea0c67498fb5f163046494bf435 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Tue, 15 Oct 2024 08:43:28 +0200 Subject: [PATCH 40/59] get from var from environ --- dds_web/commands.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dds_web/commands.py b/dds_web/commands.py index 6550a8679..52111f1a5 100644 --- a/dds_web/commands.py +++ b/dds_web/commands.py @@ -842,7 +842,7 @@ def monthly_usage(): ) # Get the instance name (DEVELOPMENT, PRODUCTION, etc.) - instance_name = flask.current_app.config.get("INSTANCE_NAME") + instance_name = os.environ.get("INSTANCE_NAME") # Email settings email_recipient: str = flask.current_app.config.get("MAIL_DDS") @@ -980,7 +980,7 @@ def send_usage(months): from dds_web.utils import current_time, page_query, send_email_with_retry # Get the instance name (DEVELOPMENT, PRODUCTION, etc.) - instance_name = flask.current_app.config.get("INSTANCE_NAME") + instance_name = os.environ.get("INSTANCE_NAME") # Email settings email_recipient: str = flask.current_app.config.get("MAIL_DDS") @@ -1149,7 +1149,7 @@ def collect_stats(): recipient: str = flask.current_app.config.get("MAIL_DDS") # Get the instance name (DEVELOPMENT, PRODUCTION, etc.) - instance_name = flask.current_app.config.get("INSTANCE_NAME") + instance_name = os.environ.get("INSTANCE_NAME") error_subject: str = "[CRONJOB]" if instance_name: # instance name can be none, so check if it is set and add it to the subject From 24544c41222107284ce247824041b537531db290 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Tue, 15 Oct 2024 08:49:07 +0200 Subject: [PATCH 41/59] get from var from environ --- dds_web/commands.py | 6 +++--- dds_web/config.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dds_web/commands.py b/dds_web/commands.py index 52111f1a5..6550a8679 100644 --- a/dds_web/commands.py +++ b/dds_web/commands.py @@ -842,7 +842,7 @@ def monthly_usage(): ) # Get the instance name (DEVELOPMENT, PRODUCTION, etc.) - instance_name = os.environ.get("INSTANCE_NAME") + instance_name = flask.current_app.config.get("INSTANCE_NAME") # Email settings email_recipient: str = flask.current_app.config.get("MAIL_DDS") @@ -980,7 +980,7 @@ def send_usage(months): from dds_web.utils import current_time, page_query, send_email_with_retry # Get the instance name (DEVELOPMENT, PRODUCTION, etc.) - instance_name = os.environ.get("INSTANCE_NAME") + instance_name = flask.current_app.config.get("INSTANCE_NAME") # Email settings email_recipient: str = flask.current_app.config.get("MAIL_DDS") @@ -1149,7 +1149,7 @@ def collect_stats(): recipient: str = flask.current_app.config.get("MAIL_DDS") # Get the instance name (DEVELOPMENT, PRODUCTION, etc.) - instance_name = os.environ.get("INSTANCE_NAME") + instance_name = flask.current_app.config.get("INSTANCE_NAME") error_subject: str = "[CRONJOB]" if instance_name: # instance name can be none, so check if it is set and add it to the subject diff --git a/dds_web/config.py b/dds_web/config.py index f40e8bdd6..d1da165f5 100644 --- a/dds_web/config.py +++ b/dds_web/config.py @@ -18,7 +18,7 @@ class Config(object): """Base config""" SITE_NAME = "Data Delivery System" # Name of the site, will appear in the website title - INSTANCE_NAME = "DEVELOPMENT" # Name of the instance, e.g. "PRODUCTION", "DEVELOPMENT", "TEST" + INSTANCE_NAME = os.environ.get("INSTANCE_NAME", "DEVELOPMENT") # Name of the instance, e.g. "PRODUCTION", "DEVELOPMENT", "TEST" SECRET_KEY = "REPLACE_THE_STRING_IN_PRODUCTION" # DB related config From a11ec8456c4ba24fdd058076bb29a6ae0c7e51eb Mon Sep 17 00:00:00 2001 From: rv0lt Date: Tue, 15 Oct 2024 08:54:14 +0200 Subject: [PATCH 42/59] black lint --- dds_web/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dds_web/config.py b/dds_web/config.py index d1da165f5..e473ec57f 100644 --- a/dds_web/config.py +++ b/dds_web/config.py @@ -18,7 +18,8 @@ class Config(object): """Base config""" SITE_NAME = "Data Delivery System" # Name of the site, will appear in the website title - INSTANCE_NAME = os.environ.get("INSTANCE_NAME", "DEVELOPMENT") # Name of the instance, e.g. "PRODUCTION", "DEVELOPMENT", "TEST" + # Name of the instance, e.g. "PRODUCTION", "DEVELOPMENT", "TEST" + INSTANCE_NAME = os.environ.get("INSTANCE_NAME", "DEVELOPMENT") SECRET_KEY = "REPLACE_THE_STRING_IN_PRODUCTION" # DB related config From f724b7e82a0a90a243a0bc73e77d477c947b6835 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Wed, 16 Oct 2024 11:22:43 +0200 Subject: [PATCH 43/59] Load env variables to flask app --- dds_web/__init__.py | 3 +++ dds_web/config.py | 5 ++--- docker-compose.yml | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/dds_web/__init__.py b/dds_web/__init__.py index 5afd64728..82db9135b 100644 --- a/dds_web/__init__.py +++ b/dds_web/__init__.py @@ -171,6 +171,9 @@ def create_app(testing=False, database_uri=None): # Initiate app object app = flask.Flask(__name__, instance_relative_config=False) + # All variables in the env that start with FLASK_* will be loaded into the app config + app.config.from_prefixed_env() + # Default development config app.config.from_object("dds_web.config.Config") diff --git a/dds_web/config.py b/dds_web/config.py index e473ec57f..2f6a52d72 100644 --- a/dds_web/config.py +++ b/dds_web/config.py @@ -16,10 +16,9 @@ class Config(object): """Base config""" - + SITE_NAME = "Data Delivery System" # Name of the site, will appear in the website title - # Name of the instance, e.g. "PRODUCTION", "DEVELOPMENT", "TEST" - INSTANCE_NAME = os.environ.get("INSTANCE_NAME", "DEVELOPMENT") + SECRET_KEY = "REPLACE_THE_STRING_IN_PRODUCTION" # DB related config diff --git a/docker-compose.yml b/docker-compose.yml index 98c287f93..2ba1bca50 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -62,6 +62,7 @@ services: - DDS_APP_CONFIG=/code/dds_web/sensitive/dds_app.cfg - FLASK_DEBUG=true - FLASK_APP=dds_web + - FLASK_INSTANCE_NAME=LOCAL_DEVELOPMENT - DB_TYPE=${DDS_DB_TYPE} # - RATELIMIT_STORAGE_URI=redis://dds_redis depends_on: From 34bc0287440072e6fc36b03422a0d76052f4693c Mon Sep 17 00:00:00 2001 From: rv0lt Date: Wed, 16 Oct 2024 11:25:29 +0200 Subject: [PATCH 44/59] new local developemtn name --- tests/test_commands.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 24e548b7d..a92ad376e 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1656,7 +1656,7 @@ def create_file_versions(project: models.Project): # Error email should be sent assert len(outbox1) == 1 assert ( - "[INVOICING CRONJOB] (DEVELOPMENT) Error in monthly-usage cronjob" + "[INVOICING CRONJOB] (LOCAL_DEVELOPMENT) Error in monthly-usage cronjob" == outbox1[-1].subject ) assert "What to do:" in outbox1[-1].body @@ -1692,7 +1692,7 @@ def create_file_versions(project: models.Project): # Error email should have been sent assert len(outbox2) == 1 assert ( - "[INVOICING CRONJOB] (DEVELOPMENT) Error in monthly-usage cronjob" + "[INVOICING CRONJOB] (LOCAL_DEVELOPMENT) Error in monthly-usage cronjob" == outbox2[-1].subject ) assert "What to do:" in outbox2[-1].body @@ -1728,7 +1728,7 @@ def create_file_versions(project: models.Project): # Email should be sent assert len(outbox3) == 1 assert ( - "[INVOICING CRONJOB] (DEVELOPMENT) Usage records available for collection" + "[INVOICING CRONJOB] (LOCAL_DEVELOPMENT) Usage records available for collection" == outbox3[-1].subject ) assert ( @@ -1752,7 +1752,7 @@ def test_monthly_usage_no_instance_name(client, cli_runner, capfd: LogCaptureFix import flask - assert flask.current_app.config.get("INSTANCE_NAME") == "DEVELOPMENT" + assert flask.current_app.config.get("INSTANCE_NAME") == "LOCAL_DEVELOPMENT" # Set the instance name to none flask.current_app.config["INSTANCE_NAME"] = None @@ -1945,7 +1945,7 @@ def run_command_and_check_output(months_to_test, start_time): # Verify output and sent email assert len(outbox) == 1 assert ( - "[SEND-USAGE CRONJOB] (DEVELOPMENT) Usage records attached in the present mail" + "[SEND-USAGE CRONJOB] (LOCAL_DEVELOPMENT) Usage records attached in the present mail" in outbox[-1].subject ) assert f"Here is the usage for the last {months_to_test} months." in outbox[-1].body @@ -2083,7 +2083,7 @@ def test_send_usage_error_csv(client, cli_runner, capfd: LogCaptureFixture): # Verify error email :- At least one email was sent assert len(outbox) == 1 assert ( - "[SEND-USAGE CRONJOB] (DEVELOPMENT) Error in send-usage cronjob" + "[SEND-USAGE CRONJOB] (LOCAL_DEVELOPMENT) Error in send-usage cronjob" == outbox[-1].subject ) assert "There was an error in the cronjob 'send-usage'" in outbox[-1].body From fb4788b7f5d2a358e9f1971d8495713f4b8becb3 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Wed, 16 Oct 2024 11:28:14 +0200 Subject: [PATCH 45/59] black lint --- dds_web/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/config.py b/dds_web/config.py index 2f6a52d72..2fd372c1b 100644 --- a/dds_web/config.py +++ b/dds_web/config.py @@ -16,7 +16,7 @@ class Config(object): """Base config""" - + SITE_NAME = "Data Delivery System" # Name of the site, will appear in the website title SECRET_KEY = "REPLACE_THE_STRING_IN_PRODUCTION" From 833b715b0f6e9bd155188940461d5d07248dfa5a Mon Sep 17 00:00:00 2001 From: rv0lt Date: Wed, 16 Oct 2024 14:18:28 +0200 Subject: [PATCH 46/59] prettier --- dds_web/config.py | 3 +-- docker-compose.yml | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dds_web/config.py b/dds_web/config.py index 2fd372c1b..be2aa4254 100644 --- a/dds_web/config.py +++ b/dds_web/config.py @@ -17,8 +17,7 @@ class Config(object): """Base config""" - SITE_NAME = "Data Delivery System" # Name of the site, will appear in the website title - + SITE_NAME = "Data Delivery System" SECRET_KEY = "REPLACE_THE_STRING_IN_PRODUCTION" # DB related config diff --git a/docker-compose.yml b/docker-compose.yml index 2ba1bca50..84649274d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,6 +63,7 @@ services: - FLASK_DEBUG=true - FLASK_APP=dds_web - FLASK_INSTANCE_NAME=LOCAL_DEVELOPMENT + - DB_TYPE=${DDS_DB_TYPE} # - RATELIMIT_STORAGE_URI=redis://dds_redis depends_on: From ef46eb69eba3306f0ffbc9acf87855b2043c9539 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Wed, 16 Oct 2024 14:19:19 +0200 Subject: [PATCH 47/59] prettier --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 84649274d..7cda25b4a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,7 +63,7 @@ services: - FLASK_DEBUG=true - FLASK_APP=dds_web - FLASK_INSTANCE_NAME=LOCAL_DEVELOPMENT - + - DB_TYPE=${DDS_DB_TYPE} # - RATELIMIT_STORAGE_URI=redis://dds_redis depends_on: From fac41e81e8fcbec2d7ba805643a0bdbd32aee381 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Revuelta?= <46089290+rv0lt@users.noreply.github.com> Date: Thu, 17 Oct 2024 09:51:18 +0200 Subject: [PATCH 48/59] Update SPRINTLOG.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ina Odén Österbo <35953392+i-oden@users.noreply.github.com> --- SPRINTLOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index 69056bcbc..1fb4b5c1c 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -440,5 +440,5 @@ _Nothing merged during this sprint_ - Update readme: backend image is published to GHCR, not DockerHub ([#1558](https://github.com/ScilifelabDataCentre/dds_web/pull/1558)) - Workflow bug fixed: PDFs (Technical Overview and Troubleshooting) were downloaded to incorrect directory([#1559](https://github.com/ScilifelabDataCentre/dds_web/pull/1559)) -- Modify the invoicing commands to send the instance name in the emails([#1561](https://github.com/ScilifelabDataCentre/dds_web/pull/1561)) - Update trivy action and add a second mirror repository to reduce TOO MANY REQUEST issue([#1560](https://github.com/ScilifelabDataCentre/dds_web/pull/1560)) +- Modify the invoicing commands to send the instance name in the emails([#1561](https://github.com/ScilifelabDataCentre/dds_web/pull/1561)) From 6b3de3484e91f682e811b0ba2237caed0e0d8254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Revuelta?= <46089290+rv0lt@users.noreply.github.com> Date: Thu, 17 Oct 2024 09:51:57 +0200 Subject: [PATCH 49/59] Update dds_web/__init__.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ina Odén Österbo <35953392+i-oden@users.noreply.github.com> --- dds_web/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dds_web/__init__.py b/dds_web/__init__.py index 82db9135b..fd87c63ca 100644 --- a/dds_web/__init__.py +++ b/dds_web/__init__.py @@ -172,6 +172,7 @@ def create_app(testing=False, database_uri=None): app = flask.Flask(__name__, instance_relative_config=False) # All variables in the env that start with FLASK_* will be loaded into the app config + # 'FLASK_' will be dropped, e.g. FLASK_TESTVAR will be loaded as TESTVAR app.config.from_prefixed_env() # Default development config From d7c15bfde1c1082ec2344c02c54778dc3f29ffd6 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 17 Oct 2024 14:04:12 +0200 Subject: [PATCH 50/59] feedback changes --- dds_web/api/superadmin_only.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/dds_web/api/superadmin_only.py b/dds_web/api/superadmin_only.py index bb7de7c0b..08fddc9a9 100644 --- a/dds_web/api/superadmin_only.py +++ b/dds_web/api/superadmin_only.py @@ -163,17 +163,14 @@ def post(self): if not motd_obj or not motd_obj.active: raise ddserr.DDSArgumentError(message=f"There is no active MOTD with ID '{motd_id}'.") - # check if sent to unit personnel only or all users - unit_personnel_only: bool = request_json.get("unit_personnel_only", False) - if not isinstance(unit_personnel_only, bool): + # check if sent to unit users only or all users + unit_only: bool = request_json.get("unit_only", False) + if not isinstance(unit_only, bool): raise ddserr.DDSArgumentError( message="The 'unit_personnel_only' argument must be a boolean." ) - if unit_personnel_only: - unit_usernames = db.session.query(models.UnitUser.username) - users_to_send = db.session.query(models.User).filter( - models.User.username.in_(unit_usernames) - ) + if unit_only: + users_to_send = db.session.query(models.UnitUser) else: users_to_send = db.session.query(models.User) From b01b858fbfc479ef1f3e0907e8bd98ac95555e25 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 17 Oct 2024 16:06:17 +0200 Subject: [PATCH 51/59] modified request param --- dds_web/api/superadmin_only.py | 4 +--- dds_web/static/swagger.yaml | 2 +- dds_web/static/swaggerv3.yaml | 2 +- tests/api/test_superadmin_only.py | 10 ++++------ tests/tests_v3/api/test_superadmin_only.py | 14 ++++++-------- 5 files changed, 13 insertions(+), 19 deletions(-) diff --git a/dds_web/api/superadmin_only.py b/dds_web/api/superadmin_only.py index 08fddc9a9..baa7a6548 100644 --- a/dds_web/api/superadmin_only.py +++ b/dds_web/api/superadmin_only.py @@ -166,9 +166,7 @@ def post(self): # check if sent to unit users only or all users unit_only: bool = request_json.get("unit_only", False) if not isinstance(unit_only, bool): - raise ddserr.DDSArgumentError( - message="The 'unit_personnel_only' argument must be a boolean." - ) + raise ddserr.DDSArgumentError(message="The 'unit_only' argument must be a boolean.") if unit_only: users_to_send = db.session.query(models.UnitUser) else: diff --git a/dds_web/static/swagger.yaml b/dds_web/static/swagger.yaml index e53f9696e..a780320a8 100644 --- a/dds_web/static/swagger.yaml +++ b/dds_web/static/swagger.yaml @@ -1447,7 +1447,7 @@ paths: motd_id: type: integer example: 1 - unit_personnel_only: + unit_only: type: boolean example: false /user/find: diff --git a/dds_web/static/swaggerv3.yaml b/dds_web/static/swaggerv3.yaml index dcd166491..8b157279b 100644 --- a/dds_web/static/swaggerv3.yaml +++ b/dds_web/static/swaggerv3.yaml @@ -1409,7 +1409,7 @@ paths: motd_id: type: integer example: 1 - unit_personnel_only: + unit_only: type: boolean example: false /user/find: diff --git a/tests/api/test_superadmin_only.py b/tests/api/test_superadmin_only.py index 51491f963..fa7dcd7e7 100644 --- a/tests/api/test_superadmin_only.py +++ b/tests/api/test_superadmin_only.py @@ -588,8 +588,8 @@ def test_send_motd_no_primary_email(client: flask.testing.FlaskClient) -> None: assert "incorrect subject" not in outbox[-1].subject -def test_send_motd_incorrect_type_unit_personnel_only(client: flask.testing.FlaskClient) -> None: - """The parameter unit_personnel_only should be a boolean""" +def test_send_motd_incorrect_type_unit_user_only(client: flask.testing.FlaskClient) -> None: + """The parameter unit_only should be a boolean""" # Authenticate token: typing.Dict = get_token(username=users["Super Admin"], client=client) @@ -608,12 +608,10 @@ def test_send_motd_incorrect_type_unit_personnel_only(client: flask.testing.Flas response: werkzeug.test.WrapperTestResponse = client.post( tests.DDSEndpoint.MOTD_SEND, headers=token, - json={"motd_id": created_motd.id, "unit_personnel_only": "some_string"}, + json={"motd_id": created_motd.id, "unit_only": "some_string"}, ) assert response.status_code == http.HTTPStatus.BAD_REQUEST - assert "The 'unit_personnel_only' argument must be a boolean." in response.json.get( - "message" - ) + assert "The 'unit_only' argument must be a boolean." in response.json.get("message") assert mock_mail_send.call_count == 0 diff --git a/tests/tests_v3/api/test_superadmin_only.py b/tests/tests_v3/api/test_superadmin_only.py index a02327f9f..1cf168359 100644 --- a/tests/tests_v3/api/test_superadmin_only.py +++ b/tests/tests_v3/api/test_superadmin_only.py @@ -575,8 +575,8 @@ def test_send_motd_no_primary_email(client: flask.testing.FlaskClient) -> None: assert "incorrect subject" not in outbox[-1].subject -def test_send_motd_incorrect_type_unit_personnel_only(client: flask.testing.FlaskClient) -> None: - """The parameter unit_personnel_only should be a boolean""" +def test_send_motd_incorrect_type_unit_only(client: flask.testing.FlaskClient) -> None: + """The parameter unit_only should be a boolean""" # Authenticate token: typing.Dict = get_token(username=users["Super Admin"], client=client) @@ -595,12 +595,10 @@ def test_send_motd_incorrect_type_unit_personnel_only(client: flask.testing.Flas response: werkzeug.test.WrapperTestResponse = client.post( tests.DDSEndpoint.MOTD_SEND, headers=token, - json={"motd_id": created_motd.id, "unit_personnel_only": "some_string"}, + json={"motd_id": created_motd.id, "unit_only": "some_string"}, ) assert response.status_code == http.HTTPStatus.BAD_REQUEST - assert "The 'unit_personnel_only' argument must be a boolean." in response.json.get( - "message" - ) + assert "The 'unit_only' argument must be a boolean." in response.json.get("message") assert mock_mail_send.call_count == 0 @@ -627,7 +625,7 @@ def test_send_motd_ok_all(client: flask.testing.FlaskClient) -> None: response: werkzeug.test.WrapperTestResponse = client.post( tests.DDSEndpoint.MOTD_SEND, headers=token, - json={"motd_id": created_motd.id, "unit_personnel_only": False}, + json={"motd_id": created_motd.id, "unit_only": False}, ) assert response.status_code == http.HTTPStatus.OK assert len(outbox) == num_users @@ -657,7 +655,7 @@ def test_send_motd_ok_unitusers(client: flask.testing.FlaskClient) -> None: response: werkzeug.test.WrapperTestResponse = client.post( tests.DDSEndpoint.MOTD_SEND, headers=token, - json={"motd_id": created_motd.id, "unit_personnel_only": True}, + json={"motd_id": created_motd.id, "unit_only": True}, ) assert response.status_code == http.HTTPStatus.OK assert len(outbox) == num_users From 75a5fcf40da28fdcf9c65832eeb8c0cf51e5d580 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 17 Oct 2024 16:07:15 +0200 Subject: [PATCH 52/59] sprintlog --- SPRINTLOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index 1fb4b5c1c..7bfb885b1 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -442,3 +442,4 @@ _Nothing merged during this sprint_ - Workflow bug fixed: PDFs (Technical Overview and Troubleshooting) were downloaded to incorrect directory([#1559](https://github.com/ScilifelabDataCentre/dds_web/pull/1559)) - Update trivy action and add a second mirror repository to reduce TOO MANY REQUEST issue([#1560](https://github.com/ScilifelabDataCentre/dds_web/pull/1560)) - Modify the invoicing commands to send the instance name in the emails([#1561](https://github.com/ScilifelabDataCentre/dds_web/pull/1561)) +- Fix the MOTD endpoint according to post merge review([#1564](https://github.com/ScilifelabDataCentre/dds_web/pull/1564)) From 4e87bd0c2fb09f9fec1d4befc95441ddc064a154 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 17 Oct 2024 16:11:19 +0200 Subject: [PATCH 53/59] fix typo --- dds_web/api/superadmin_only.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dds_web/api/superadmin_only.py b/dds_web/api/superadmin_only.py index baa7a6548..3cc435079 100644 --- a/dds_web/api/superadmin_only.py +++ b/dds_web/api/superadmin_only.py @@ -207,7 +207,7 @@ def post(self): utils.send_email_with_retry(msg=msg, obj=conn) return_msg = f"MOTD '{motd_id}' has been " - if unit_personnel_only: + if unit_only: return_msg += "sent to unit personnel only." else: return_msg += "sent to all users." From 4df4a04b467b4426a2d1ea78c1a67e26939beab9 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 17 Oct 2024 16:42:55 +0200 Subject: [PATCH 54/59] fix tests --- tests/api/test_superadmin_only.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/api/test_superadmin_only.py b/tests/api/test_superadmin_only.py index fa7dcd7e7..59286f7cc 100644 --- a/tests/api/test_superadmin_only.py +++ b/tests/api/test_superadmin_only.py @@ -611,7 +611,9 @@ def test_send_motd_incorrect_type_unit_user_only(client: flask.testing.FlaskClie json={"motd_id": created_motd.id, "unit_only": "some_string"}, ) assert response.status_code == http.HTTPStatus.BAD_REQUEST - assert "The 'unit_only' argument must be a boolean." in response.json.get("message") + assert "The 'unit_only' argument must be a boolean." in response.json.get( + "message" + ) assert mock_mail_send.call_count == 0 @@ -638,7 +640,7 @@ def test_send_motd_ok_all(client: flask.testing.FlaskClient) -> None: response: werkzeug.test.WrapperTestResponse = client.post( tests.DDSEndpoint.MOTD_SEND, headers=token, - json={"motd_id": created_motd.id, "unit_personnel_only": False}, + json={"motd_id": created_motd.id, "unit_only": False}, ) assert response.status_code == http.HTTPStatus.OK assert len(outbox) == num_users @@ -668,7 +670,7 @@ def test_send_motd_ok_unitusers(client: flask.testing.FlaskClient) -> None: response: werkzeug.test.WrapperTestResponse = client.post( tests.DDSEndpoint.MOTD_SEND, headers=token, - json={"motd_id": created_motd.id, "unit_personnel_only": True}, + json={"motd_id": created_motd.id, "unit_only": True}, ) assert response.status_code == http.HTTPStatus.OK assert len(outbox) == num_users From 9fb4bf3b8b38ae0bd36681efcf3f27a32d3afb58 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Thu, 17 Oct 2024 16:45:29 +0200 Subject: [PATCH 55/59] black --- tests/api/test_superadmin_only.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/api/test_superadmin_only.py b/tests/api/test_superadmin_only.py index 59286f7cc..487a3efad 100644 --- a/tests/api/test_superadmin_only.py +++ b/tests/api/test_superadmin_only.py @@ -611,9 +611,7 @@ def test_send_motd_incorrect_type_unit_user_only(client: flask.testing.FlaskClie json={"motd_id": created_motd.id, "unit_only": "some_string"}, ) assert response.status_code == http.HTTPStatus.BAD_REQUEST - assert "The 'unit_only' argument must be a boolean." in response.json.get( - "message" - ) + assert "The 'unit_only' argument must be a boolean." in response.json.get("message") assert mock_mail_send.call_count == 0 From dbc6b6b76e63f605ddc6d53501f1a616227d81d6 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Mon, 21 Oct 2024 09:31:31 +0200 Subject: [PATCH 56/59] changelog --- CHANGELOG.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1d635628a..22d87a444 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,19 @@ Changelog ========== +.. _2.8.1: + +2.8.1 - 2024-10-23 +~~~~~~~~~~~~~~~~~~~~~~~ + +- New features: + - warning_level option when a unit is created defaults to 0.8. + - Add option to MOTD endpoint to send an email to unit users only. + - Modify the invoicing commands to send the instance name in the emails. +- Bugs fixed: + - Update +- Update readme to indicate that the backend image is published to GGCR, not DockerHub. + .. _2.8.0: 2.8.0 - 2024-09-24 From ddfbbc77f6f9bc7f38797180c06f420c7b0fd75e Mon Sep 17 00:00:00 2001 From: rv0lt Date: Mon, 21 Oct 2024 09:32:59 +0200 Subject: [PATCH 57/59] new version --- dds_web/version.py | 2 +- tests/test_version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dds_web/version.py b/dds_web/version.py index 91ebb8d13..dcd4a7562 100644 --- a/dds_web/version.py +++ b/dds_web/version.py @@ -1,3 +1,3 @@ # Do not do major version upgrade during 2024. # If mid or minor version reaches 9, continue with 10, 11 etc etc. -__version__ = "2.8.0" +__version__ = "2.8.1" diff --git a/tests/test_version.py b/tests/test_version.py index ff0e1562e..2ef1533b8 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -2,4 +2,4 @@ def test_version(): - assert version.__version__ == "2.8.0" + assert version.__version__ == "2.8.1" From 75882d3dc2c56685311851ce872b592ab5b73388 Mon Sep 17 00:00:00 2001 From: rv0lt Date: Mon, 21 Oct 2024 09:35:22 +0200 Subject: [PATCH 58/59] sprintlog --- SPRINTLOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/SPRINTLOG.md b/SPRINTLOG.md index 7bfb885b1..18638a870 100644 --- a/SPRINTLOG.md +++ b/SPRINTLOG.md @@ -443,3 +443,4 @@ _Nothing merged during this sprint_ - Update trivy action and add a second mirror repository to reduce TOO MANY REQUEST issue([#1560](https://github.com/ScilifelabDataCentre/dds_web/pull/1560)) - Modify the invoicing commands to send the instance name in the emails([#1561](https://github.com/ScilifelabDataCentre/dds_web/pull/1561)) - Fix the MOTD endpoint according to post merge review([#1564](https://github.com/ScilifelabDataCentre/dds_web/pull/1564)) +- New version & changelog([#1565](https://github.com/ScilifelabDataCentre/dds_web/pull/1565)) From 545ae60a9f6e0803d17a5897f815b03643364a9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Revuelta?= <46089290+rv0lt@users.noreply.github.com> Date: Mon, 21 Oct 2024 10:03:05 +0200 Subject: [PATCH 59/59] Update CHANGELOG.rst Co-authored-by: Valentin Georgiev --- CHANGELOG.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 22d87a444..62c932f5c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,9 +10,7 @@ Changelog - warning_level option when a unit is created defaults to 0.8. - Add option to MOTD endpoint to send an email to unit users only. - Modify the invoicing commands to send the instance name in the emails. -- Bugs fixed: - - Update -- Update readme to indicate that the backend image is published to GGCR, not DockerHub. +- Documentation: Update readme to indicate that the backend image is published to GGCR, not DockerHub. .. _2.8.0: