Merge pull request #1582 from ScilifelabDataCentre/dev

New release: December 18th 2024

.github/workflows/codeql-analysis.yml

@@ -51,11 +51,11 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         with:
           languages: ${{ matrix.language }}
           # If you wish to specify custom queries, you can do so here or in a config file.
@@ -66,7 +66,7 @@ jobs:
       # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
+        uses: github/codeql-action/autobuild@v3
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 https://git.io/JvXDl
@@ -80,4 +80,4 @@ jobs:
       #   make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3

.github/workflows/docker-compose-tests.yml

@@ -19,7 +19,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Run tests against database container
         run: docker compose -f docker-compose.yml -f tests/docker-compose-test.yml up --build --exit-code-from backend
@@ -31,7 +31,7 @@ jobs:
           ## If no one connects after 5 minutes, shut down server.
           wait-timeout-minutes: 5
 
-      - uses: codecov/codecov-action@v3
+      - uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./coverage/report.xml

.github/workflows/lint-yaml.yml

@@ -13,7 +13,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: yaml-lint
         uses: ibiqlik/action-yamllint@v3
         with:

.github/workflows/prettier.yml

@@ -9,9 +9,9 @@ jobs:
   Prettier:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
 
       - name: Install Prettier
         run: npm install -g prettier

.github/workflows/python-black.yml

@@ -15,7 +15,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: psf/black@stable
         with:
           version: ${{ env.INPUT_VERSION }}

.github/workflows/snyk-scan.yml

@@ -30,7 +30,7 @@ jobs:
           command: code test
           args: --sarif-file-output=snyk.sarif
       - name: Upload result to GitHub Code Scanning
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: snyk.sarif
           category: snyk

.github/workflows/trivy-scan-branch.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@master

.github/workflows/trivy-scheduled-dev.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Ensure lowercase name
         run: echo REPOSITORY_OWNER=$(echo ${{ github.repository_owner }} | tr "[:upper:]" "[:lower:]") >> $GITHUB_ENV

.github/workflows/trivy-scheduled-master.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: master
 

CHANGELOG.rst

@@ -1,6 +1,24 @@
 Changelog
 ==========
 
+.. _2.9.0:
+
+2.9.0 - 2024-12-18
+~~~~~~~~~~~~~~~~~~
+
+- New features:
+    - Delivery directory names now include 'Upload' or 'Download' for clarity.
+    - Monitor usage now sends warnings to the affected unit and Data Centre when approaching the allocated quota.
+- Bugs fixed:
+    - Improved error message for downloads after a password reset.
+    - Fixed the dds ls --tree command.
+    - Pinned mariadb-client version in backend Dockerfile for stability.
+    - Resolved Node.js vulnerabilities with npm audit fix.
+- Logging:
+    - Removed invalid token exceptions from logs.
+    - Logged usernames for password resets.
+    - Filtered out MaintenanceModeException from logs.
+
 .. _2.8.1:
 
 2.8.1 - 2024-10-23

Dockerfiles/backend.Dockerfile

@@ -49,7 +49,12 @@ ENV PYTHONPATH /code
 ###################
 FROM base as test
 RUN pip3 install -r /code/tests/requirements-test.txt
-RUN apk add mariadb-client
+
+# The version of mariadb-client should match the version of the mariadb server
+# Because of how alpine works, this is how to pin a version. However, it can break if this branch is removed from alpine
+# https://superuser.com/questions/1055060/how-to-install-a-specific-package-version-in-alpine
+# https://pkgs.alpinelinux.org/packages?name=mariadb-client&branch=v3.19&repo=&arch=x86_64&origin=&flagged=&maintainer=
+RUN apk add mariadb-client=~10.11 --repository https://dl-cdn.alpinelinux.org/alpine/v3.19/main/
 
 # Switch to the user
 USER $USERNAME
@@ -60,7 +65,7 @@ USER $USERNAME
 FROM node:18 as nodebuilder
 COPY ./dds_web/static /build
 WORKDIR /build
-RUN npm install -g npm@latest --quiet
+RUN npm install -g npm@10.9.2 --quiet
 RUN npm install --quiet
 RUN npm run css
 

Dockerfiles/nodebuilder.Dockerfile

@@ -1,6 +1,6 @@
 FROM node:18
 RUN mkdir /build
 WORKDIR /build
-RUN npm install -g npm@latest --quiet
+RUN npm install -g npm@10.9.2 --quiet
 RUN echo "npm install --quiet && npm run watch" > /runner.sh
 CMD ["sh", "/runner.sh"]

SPRINTLOG.md

@@ -430,11 +430,11 @@ _Nothing merged during this sprint_
 
 - Flask command to update unit quotas ([#1551](https://github.com/ScilifelabDataCentre/dds_web/pull/1551))
 - Bump python base image to 3.12 and related libraries in both web and client([#1548](https://github.com/ScilifelabDataCentre/dds_web/pull/1548))
-- Warning_level option defaults to 0.8([#1557](https://github.com/ScilifelabDataCentre/dds_web/pull/1557))
 
 # 2024-09-24 - 2024-10-04
 
 - Add option to motd command for sending to unit users only([#1552](https://github.com/ScilifelabDataCentre/dds_web/pull/1552))
+- Warning_level option defaults to 0.8([#1557](https://github.com/ScilifelabDataCentre/dds_web/pull/1557))
 
 # 2024-10-07 - 2024-10-18
 
@@ -444,3 +444,31 @@ _Nothing merged during this sprint_
 - Modify the invoicing commands to send the instance name in the emails([#1561](https://github.com/ScilifelabDataCentre/dds_web/pull/1561))
 - Fix the MOTD endpoint according to post merge review([#1564](https://github.com/ScilifelabDataCentre/dds_web/pull/1564))
 - New version & changelog([#1565](https://github.com/ScilifelabDataCentre/dds_web/pull/1565))
+
+# 2024-10-21 - 2024-11-01
+
+- Workflow: Bump GitHub checkout action to v4 ([#1556](https://github.com/ScilifelabDataCentre/dds_web/pull/1556))
+- Workflow: CodeQL action version(s) bumped to v3 ([#1569](https://github.com/ScilifelabDataCentre/dds_web/pull/1569))
+- Workflow: Setup-node, codecov and upload-sarif action versions bumped to v4, v4 and v3, respectively ([#1570](https://github.com/ScilifelabDataCentre/dds_web/pull/1570))
+
+# 2024-11-04 - 2024-11-15
+
+- Removed exception for invalid token to simplify logging and reduce unnecessary error entries ([#1572](https://github.com/ScilifelabDataCentre/dds_web/pull/1572))
+
+# 2024-11-18 – 2024-11-29
+
+- Logging: Add which user name reset password ([https://github.com/ScilifelabDataCentre/dds_web/pull/1574](https://github.com/ScilifelabDataCentre/dds_web/pull/1574))
+
+# 2024-12-02 – 2024-12-13
+
+- Change the error raised upon attempt to download data after a password reset to an AuthenticationError to avoid getting an alert ([#1571](https://github.com/ScilifelabDataCentre/dds_web/pull/1571))
+- Filter out the MaintenanceModeException from the logs ([#1573](https://github.com/ScilifelabDataCentre/dds_web/pull/1573))
+- Bugfix: Quick and dirty change to prevent `dds ls --tree` from failing systematically ([#1575](https://github.com/ScilifelabDataCentre/dds_web/pull/1575))
+- Update backend Dockerfile to pin a fixed version of mariadb-client ([#1581](https://github.com/ScilifelabDataCentre/dds_web/pull/1581))
+- Update documentation regarding 'Upload' or 'Download' added to end of delivery directory name depending on command ([#1580](https://github.com/ScilifelabDataCentre/dds_web/pull/1580))
+- Modify the monitor usage command to send warning to the affected unit as well as Data Centre([#1562](https://github.com/ScilifelabDataCentre/dds_web/pull/1562))
+- Run npm audit fix to solve node cve's ([#1577](https://github.com/ScilifelabDataCentre/dds_web/pull/1577)
+
+# 2024-12-16 - 2024-12-20
+
+- New version: 2.9.0 ([#1584](https://github.com/ScilifelabDataCentre/dds_web/pull/1584))

dds_web/__init__.py

@@ -68,6 +68,20 @@
 ####################################################################################################
 
 
+class FilterMaintenanceExc(logging.Filter):
+
+    def filter(record):
+        """
+        Filters log records to exclude those with MaintenanceOngoingException.
+        Returns:
+            bool: True if the log record should be logged, False if it should be filtered out.
+        """
+        from dds_web.errors import MaintenanceOngoingException
+
+        # Check if the log record does not have an exception or if the exception is not MaintenanceOngoingException
+        return record.exc_info is None or record.exc_info[0] != MaintenanceOngoingException
+
+
 def setup_logging(app):
     """Setup loggers"""
 
@@ -164,6 +178,9 @@ def action_wrapper(_, __, event_dict):
         cache_logger_on_first_use=True,
     )
 
+    # Add custom filter to the logger
+    logging.getLogger("general").addFilter(FilterMaintenanceExc)
+
 
 def create_app(testing=False, database_uri=None):
     try:

dds_web/api/files.py

@@ -324,7 +324,7 @@ def old_get(self, project):
         show_size = extra_args.get("show_size")
 
         # Check if to get from root or folder
-        subpath = extra_args.get("subpath", ".").rstrip(os.sep)
+        subpath = (extra_args.get("subpath") or ".").rstrip(os.sep)
         subpath = "." if subpath == "" else subpath
 
         return self.get_files_folders(project, subpath, show_size)

dds_web/api/project.py

@@ -592,7 +592,7 @@ def get(self):
         """Get private key from database."""
         # Verify project ID and access
         project = project_schemas.ProjectRequiredSchema().load(flask.request.args)
-
+        dds_web.utils.verify_project_user_key(project=project)
         flask.current_app.logger.debug("Getting the private key.")
 
         return flask.jsonify(

dds_web/commands.py

@@ -1269,7 +1269,7 @@ def monitor_usage():
     import dds_web.utils
 
     # Email settings
-    recipient: str = flask.current_app.config.get("MAIL_DDS")
+    dds_contact: str = flask.current_app.config.get("MAIL_DDS")
     default_subject: str = "DDS: Usage quota warning!"
 
     # Run task
@@ -1305,15 +1305,17 @@ def monitor_usage():
         # Email if the unit is using more
         if perc_used_decimal > warn_after:
             # Email settings
+            unit_contact: str = unit.contact_email
             message: str = (
-                "A SciLifeLab Unit is approaching the allocated data quota.\n"
-                f"Affected unit: {unit.name}\n"
+                "Your unit is approaching the allocated data quota (see details below).\n\n"
+                f"NB! If you would like to increase or decrease the allocated quota ('Quota') or the level after which you receive this email ('Warning level'), the technical contact person for your unit must send a request to {dds_contact}.\n"
+                f"Unit name: {unit.name}\n"
                 f"{info_string}"
             )
             flask.current_app.logger.info(message)
             msg: flask_mail.Message = flask_mail.Message(
                 subject=default_subject,
-                recipients=[recipient],
+                recipients=[unit_contact, dds_contact],
                 body=message,
             )
             dds_web.utils.send_email_with_retry(msg=msg)

dds_web/database/models.py

@@ -187,7 +187,7 @@ class Unit(db.Model):
     public_id = db.Column(db.String(50), unique=True, nullable=False)
     name = db.Column(db.String(255), unique=True, nullable=False)
     external_display_name = db.Column(db.String(255), unique=False, nullable=False)
-    contact_email = db.Column(db.String(255), unique=False, nullable=True)
+    contact_email = db.Column(db.String(255), unique=False, nullable=False)
     internal_ref = db.Column(db.String(50), unique=True, nullable=False)
 
     # Safespring storage

dds_web/security/auth.py

@@ -278,7 +278,7 @@ def __verify_general_token(token):
         # ValueError is raised when the token doesn't look right (for example no periods)
         # jwcryopto.common.JWException is the base exception raised by jwcrypto,
         # and is raised when the token is malformed or invalid.
-        flask.current_app.logger.exception(e)
+        flask.current_app.logger.warning(f"Error with Token operation: {type(e).__name__}")
         raise AuthenticationError(message="Invalid token") from e
 
     expiration_time = data.get("exp")