Merge pull request #1391 from ScilifelabDataCentre/dev

Release to fix security issues

.github/pull_request_template.md

@@ -105,6 +105,8 @@ _Add information on additional changes required for the PR changes to work, both
 - **Prettier**: General code formatter. Our use case: MD and yaml mainly.
   Run `npx prettier --write .` locally to execute formatting.
   - [ ] Passed
+- **Yamllint**: Linting of yaml files.
+  - [ ] Passed
 - **Tests**: Pytest to verify that functionality works as expected.
   - [ ] New tests added
   - [ ] No new tests

.github/workflows/codeql-analysis.yml

@@ -18,8 +18,8 @@
 # We have attempted to detect the languages in your repository. Please check
 # the `language` matrix defined below to confirm you have the correct set of
 # supported CodeQL languages.
-#
-name: "CodeQL"
+---
+name: CodeQL
 
 on:
   push:

.github/workflows/docker-compose-tests.yml

@@ -1,8 +1,8 @@
 # Pytest
 # Runs all test in the dds_web by executing the docker compose yml files for testing.
 # The actual pytest command is not in this file.
-
-name: Tests
+---
+name: Pytest
 
 on:
   push:

.github/workflows/lint-yaml.yml

@@ -0,0 +1,24 @@
+# Yamtllint: Lint Yaml files.
+# A linter for YAML files. Checks for:
+# - syntax validity
+# - weirdnesses like key repetition and cosmetic problems such as:
+#   - lines length
+#   - trailing spaces
+#   - indentation
+# etc.
+---
+name: Yamllint
+on: [push]
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: yaml-lint
+        uses: ibiqlik/action-yamllint@v3
+        with:
+          config_data: |
+            extends: default
+            rules:
+              line-length:
+                max: 120

.github/workflows/prettier.yml

@@ -1,8 +1,8 @@
-# Prettier
+# Prettier: Format code.
 # Prettier is a code formatter. We mostly use it for the markdown files.
 # https://prettier.io/
-
-name: Lint with Prettier
+---
+name: Prettier
 on: [push, pull_request]
 
 jobs:

.github/workflows/publish_and_trivyscan.yml

@@ -12,7 +12,7 @@
 # https://github.com/aquasecurity/trivy
 # ---------------------------------------
 # Publish Docker Image to GHCR and run Trivy Security Scan
-name: GHCR and Trivy Scan
+name: Publish to GHCR (+ Trivy scan)
 on:
   workflow_dispatch:
     branches: [dev]

.github/workflows/python-black.yml

@@ -1,9 +1,10 @@
-# Lint Python - Black
+# Black: Python linter
 # This action lints python using black - a python code formatter.
 # https://github.com/psf/black
 # This does not format the code, this only detects and informs on issues.
 # To format with black, run `black .` locally in the repository.
-name: Black - Lint Python
+---
+name: Black
 
 on: [push, pull_request]
 

.github/workflows/snyk-scan.yml

@@ -1,17 +1,8 @@
-# Snyk IaC:
-# A sample workflow which checks out your Infrastructure as Code Configuration files,
-# such as Kubernetes, Helm & Terraform and scans them for any security issues.
+# Snyk: Scan for for vulnerabilities in your Python code and dependencies
 # The results are then uploaded to GitHub Security Code Scanning
-#
-# For more examples, including how to limit scans to only high-severity issues
-# and fail PR checks, see https://github.com/snyk/actions/
-# -----------------------------------------------------------
-# This workflow uses actions that are not certified by GitHub.
-# They are provided by a third-party and are governed by
-# separate terms of service, privacy policy, and support
-# documentation.
-# ------------------------------------------------------------
-name: Snyk IaC
+# https://github.com/snyk/actions/
+---
+name: Snyk
 
 on:
   push:

.github/workflows/trivy-scan-branch.yml

@@ -0,0 +1,37 @@
+# Trivy: Scan of current branch
+# Trivy is a comprehensive and versatile security scanner.
+# Trivy has scanners that look for security issues, and targets where it can find those issues.
+# https://github.com/aquasecurity/trivy
+# ---------------------------------
+name: Trivy - branch scan
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+      - dev
+jobs:
+  scan:
+    permissions:
+      contents: read
+      security-events: write
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          scan-type: "fs"
+          ignore-unfixed: true
+          format: "sarif"
+          output: "trivy-results.sarif"
+          severity: "CRITICAL,HIGH"
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: "trivy-results.sarif"
+          category: trivy

.github/workflows/trivy.yml → .github/workflows/trivy-scheduled-dev.yml

@@ -1,17 +1,16 @@
-# Trivy:
+# Trivy: Scan remote image on GHCR on schedule
 # Trivy (pronunciation) is a comprehensive and versatile security scanner.
-# Trivy has scanners that look for security issues, and targets where it can find those issues.
+# Trivy has scanners that look for security issues,
+# and targets where it can find those issues.
 # https://github.com/aquasecurity/trivy
 # ---------------------------------
-name: trivy
+name: Trivy - ghcr image scan - dev
 on:
   workflow_dispatch:
     branches:
-      - master
       - dev
   schedule:
-    # - cron: "*/5 * * * *"
-    - cron: "0 7,13 * * *"
+    - cron: "0 9,12,15 * * *"
 jobs:
   scan:
     permissions:
@@ -26,16 +25,16 @@ jobs:
       - name: Ensure lowercase name
         run: echo REPOSITORY_OWNER=$(echo ${{ github.repository_owner }} | tr "[:upper:]" "[:lower:]") >> $GITHUB_ENV
 
-      - name: Run Trivy vulnerability scanner
+      - name: Run Trivy on latest dev image
         uses: aquasecurity/[email protected]
         with:
-          image-ref: "ghcr.io/${{ env.REPOSITORY_OWNER }}/dds-backend:latest"
+          image-ref: "ghcr.io/${{ env.REPOSITORY_OWNER }}/dds-backend:dev"
           format: "sarif"
-          output: "trivy-results.sarif"
+          output: "trivy-results-dev.sarif"
           severity: "CRITICAL,HIGH"
 
-      - name: Upload Trivy scan results to GitHub Security tab
+      - name: Upload Trivy scan results to dev branch GitHub Security tab
         uses: github/codeql-action/upload-sarif@v2
         with:
-          sarif_file: "trivy-results.sarif"
-          category: trivy
+          sarif_file: "trivy-results-dev.sarif"
+          category: trivy-dev

.github/workflows/trivy-scheduled-master.yml

@@ -0,0 +1,42 @@
+# Trivy: Scan remote image on GHCR on schedule
+# Trivy (pronunciation) is a comprehensive and versatile security scanner.
+# Trivy has scanners that look for security issues,
+# and targets where it can find those issues.
+# https://github.com/aquasecurity/trivy
+# ---------------------------------
+name: Trivy - ghcr image scan - master
+on:
+  workflow_dispatch:
+    branches:
+      - master
+  schedule:
+    - cron: "0 7,15 * * *"
+jobs:
+  scan:
+    permissions:
+      contents: read
+      security-events: write
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          ref: master
+
+      - name: Ensure lowercase name
+        run: echo REPOSITORY_OWNER=$(echo ${{ github.repository_owner }} | tr "[:upper:]" "[:lower:]") >> $GITHUB_ENV
+
+      - name: Run Trivy on latest release image
+        uses: aquasecurity/[email protected]
+        with:
+          image-ref: "ghcr.io/${{ env.REPOSITORY_OWNER }}/dds-backend:latest"
+          format: "sarif"
+          output: "trivy-results-master.sarif"
+          severity: "CRITICAL,HIGH"
+
+      - name: Upload Trivy scan results to master branch GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: "trivy-results-master.sarif"
+          category: trivy-master

.gitpod.yml

@@ -43,7 +43,8 @@ vscode:
     - ms-azuretools.vscode-docker
     - ms-python.python
     - esbenp.prettier-vscode # Linting and style checking
-    - Gruntfuggly.todo-tree # Display TODO and FIXME in a tree view in the activity bar
+    # Display TODO and FIXME in a tree view in the activity bar
+    - Gruntfuggly.todo-tree
 
 github:
   prebuilds:
@@ -57,7 +58,7 @@ github:
     pullRequestsFromForks: true
     # add a check to pull requests (defaults to true)
     addCheck: true
-    # add a "Review in Gitpod" button as a comment to pull requests (defaults to false)
+    # add a "Review in Gitpod" button as a comment to PR (default: false)
     addComment: false
-    # add a "Review in Gitpod" button to the pull request's description (defaults to false)
+    # add a "Review in Gitpod" button to the PR's description (default: false)
     addBadge: true

.prettierrc.yaml

@@ -1 +1,2 @@
+---
 printWidth: 120

CHANGELOG.md

@@ -200,3 +200,23 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe
 - Workflow: Add option to publish dev image manually ([#1376](https://github.com/ScilifelabDataCentre/dds_web/pull/1376))
 - Bug: Add value to `Unit.warning_level` for existing units ([#1378](https://github.com/ScilifelabDataCentre/dds_web/pull/1379))
 - Workflow: Add option to run trivy on dev and master branches manually ([#1380](https://github.com/ScilifelabDataCentre/dds_web/pull/1380))
+
+## Sprint (2023-02-03 - 2023-02-17)
+
+- Workflow: Scan with yamllint ([#1385](https://github.com/ScilifelabDataCentre/dds_web/pull/1385))
+
+## Sprint (2023-02-17 - 2023-03-03)
+
+- Move Rotating log file maximum size and backup count to config variables ([#1388](https://github.com/ScilifelabDataCentre/dds_web/pull/1388))
+- Workflow: Scan branch with trivy ([#1377](https://github.com/ScilifelabDataCentre/dds_web/pull/1377))
+- Dependencies bumped ([#1387](https://github.com/ScilifelabDataCentre/dds_web/pull/1387))
+  - Werkzeug: 2.0.3 --> 2.2.3 (CVE-2023-25577)
+  - MarkupSafe: 2.0.1 --> 2.1.1 (prior bumps)
+  - Flask-Login: 0.5.0 --> 0.6.2 (prior bumps)
+  - Flask: 2.0.2 --> 2.0.3 (prior bumps)
+  - Cryptography: 36.0.1 --> 39.0.1 (CVE-2023-0286)
+- Npm vulnerability fixed: CVE-2022-25881 ([#1390](https://github.com/ScilifelabDataCentre/dds_web/pull/1390))
+- Logging: Configure action-logging to wrap json with parent key "action" ([https://github.com/ScilifelabDataCentre/dds_web/pull/1393](https://github.com/ScilifelabDataCentre/dds_web/pull/1393))
+- Workflow: Schedule trivy scan for both dev images and latest release ([#1392](https://github.com/ScilifelabDataCentre/dds_web/pull/1392))
+- Improve logging of delete-invites flask command ([#1386](https://github.com/ScilifelabDataCentre/dds_web/pull/1386))
+- Workflow: Schedule trivy scan for dev and latest separately ([#1395](https://github.com/ScilifelabDataCentre/dds_web/pull/1395))

codecov.yml

@@ -1,3 +1,4 @@
+---
 coverage:
   range: 60..90
   round: down
@@ -7,6 +8,5 @@ coverage:
       default:
         threshold: 5%
     patch: off
-
 ignore:
   - "dds_web/development/*"

dds_web/__init__.py

@@ -71,6 +71,9 @@
 def setup_logging(app):
     """Setup loggers"""
 
+    def action_wrapper(_, __, event_dict):
+        return {"action": event_dict}
+
     dictConfig(
         {
             "version": 1,
@@ -85,14 +88,14 @@ def setup_logging(app):
                     "class": "logging.handlers.RotatingFileHandler",
                     "filename": pathlib.Path(app.config.get("LOGS_DIR")) / pathlib.Path("dds.log"),
                     "formatter": "general",
-                    "maxBytes": 0x100000,
-                    "backupCount": 15,
+                    "maxBytes": app.config.get("LOG_MAX_SIZE"),
+                    "backupCount": app.config.get("LOG_BACKUP_COUNT"),
                 },
                 "actions": {
                     "level": logging.INFO,
                     "class": "logging.handlers.RotatingFileHandler",
-                    "maxBytes": 0x100000,
-                    "backupCount": 15,
+                    "maxBytes": app.config.get("LOG_MAX_SIZE"),
+                    "backupCount": app.config.get("LOG_BACKUP_COUNT"),
                     "filename": pathlib.Path(app.config.get("LOGS_DIR"))
                     / pathlib.Path("actions.log"),
                     "formatter": "default",
@@ -142,6 +145,9 @@ def setup_logging(app):
             structlog.processors.format_exc_info,
             # If some value is in bytes, decode it to a unicode str.
             structlog.processors.UnicodeDecoder(),
+            # Wrap each log row under the parent key "action": {"action": <json dict rendered below>}
+            # Why: To enable filtering of the action logs
+            action_wrapper,
             # Render the final event dict as JSON.
             structlog.processors.JSONRenderer(),
         ],

dds_web/api/dds_decorators.py

@@ -55,7 +55,7 @@ def handle_error(*args, **kwargs):
 def json_required(func):
     @functools.wraps(func)
     def verify_json(*args, **kwargs):
-        if not flask.request.json:
+        if not flask.request.get_json(silent=True):
             raise MissingJsonError(message="Required data missing from request!")
 
         return func(*args, **kwargs)