Merge branch 'development' into HYP-184

.github/workflows/requirements-update.yml

@@ -0,0 +1,60 @@
+name: Requirements Update
+
+on:
+  schedule:
+    - cron: '0 12 * * 1' # runs at 12:00 UTC on Mondays
+  workflow_dispatch:
+
+jobs:
+
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v4
+        with:
+          only-labels: dependencies,automated pr
+          stale-pr-message: 'This PR is stale because it has been open 7 days with no activity. Remove stale label or comment or this will be closed in 7 days.'
+          close-pr-message: 'This PR was closed because it has been stalled for 7 days with no activity.'
+          days-before-pr-stale: 7
+          days-before-pr-close: 7
+          delete-branch: true
+
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        ref: development
+
+    - name: Setup python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.10'
+
+    - name: Install dev Python packages
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r dev-requirements.txt
+
+    - name: Check for pip-tools upgrades
+      run: |
+        pip-compile --generate-hashes \
+          --allow-unsafe \
+          --upgrade \
+          --output-file requirements.txt requirements.in
+
+    - name: Create Pull Request
+      uses: peter-evans/create-pull-request@v3
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        base: development
+        branch: requirements-updates
+        branch-suffix: timestamp
+        delete-branch: true
+        commit-message: "fix(requirements): Updated Python requirements"
+        title: 'Python Requirements Updates'
+        body: >
+          This PR is auto-generated by Github Actions job [requirements-update].
+        labels: dependencies, automated pr

.github/workflows/scan.yml

@@ -0,0 +1,67 @@
+name: Scan
+
+on:
+  push:
+    branches: [ master, development ]
+  pull_request:
+    branches: [ master, development ]
+  schedule:
+    - cron: '0 12 * * 1' # runs at 12:00 UTC on Mondays
+  workflow_dispatch:
+
+jobs:
+
+  scan:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKER_HUB_USERNAME }}
+        password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+
+    - name: Set image name
+      id: setimagename
+      run: |
+          echo "Image name: $GITHUB_REPOSITORY:$GITHUB_SHA"
+          echo "::set-output name=imagename::$GITHUB_REPOSITORY:$GITHUB_SHA"
+
+    - name: Build the image
+      id: buildimage
+      uses: docker/build-push-action@v2
+      with:
+        context: ./
+        file: ./Dockerfile
+        push: false
+        tags: ${{ steps.setimagename.outputs.imagename }}
+
+    - name: Check whether container scanning should be enabled
+      id: checkcontainerscanning
+      env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+      run: |
+          echo "Enable container scanning: ${{ env.SNYK_TOKEN != '' }}"
+          echo "::set-output name=enabled::${{ env.SNYK_TOKEN != '' }}"
+
+    - name: Run Snyk to check Docker image for vulnerabilities
+      uses: snyk/actions/docker@master
+      if: steps.checkcontainerscanning.outputs.enabled == 'true'
+      continue-on-error: true
+      env:
+        SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+      with:
+        image: ${{ steps.setimagename.outputs.imagename }}
+        args: --file=Dockerfile
+
+    - name: Upload result to GitHub Code Scanning
+      uses: github/codeql-action/upload-sarif@v1
+      if: steps.checkcontainerscanning.outputs.enabled == 'true'
+      with:
+        sarif_file: snyk.sarif

.github/workflows/test.yml

@@ -0,0 +1,40 @@
+name: Test
+
+on:
+  push:
+    branches: [ master, development ]
+  pull_request:
+    branches: [ master, development ]
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKER_HUB_USERNAME }}
+        password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+
+    - name: Set image name
+      id: setimagename
+      run: |
+          echo "Image name: $GITHUB_REPOSITORY:$GITHUB_SHA"
+          echo "::set-output name=imagename::$GITHUB_REPOSITORY:$GITHUB_SHA"
+
+    - name: Build the image
+      id: buildimage
+      uses: docker/build-push-action@v2
+      with:
+        context: ./
+        file: ./Dockerfile
+        push: false
+        tags: ${{ steps.setimagename.outputs.imagename }}

.gitignore

@@ -1,8 +1,9 @@
-app/assets/*
-*.pyc
-*/.DS_Store
-.DS_Store
-*.log
-app/db.sqlite3
-app/hypatio/local_settings.py
+app/assets/*
+*.pyc
+*/.DS_Store
+.DS_Store
+*.log
+app/db.sqlite3
+app/hypatio/local_settings.py
 .vscode/settings.json
+backup

.pre-commit-config.yaml

@@ -0,0 +1,20 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks.git
+    rev: v4.2.0
+    hooks:
+    - id: trailing-whitespace
+    - id: mixed-line-ending
+    - id: check-byte-order-marker
+    - id: check-merge-conflict
+    - id: detect-aws-credentials
+  - repo: https://github.com/jazzband/pip-tools
+    rev: 6.8.0
+    hooks:
+      - id: pip-compile
+        name: pip-compile dev-requirements.in
+        args: [dev-requirements.in, --upgrade, --generate-hashes, --allow-unsafe, --output-file, dev-requirements.txt]
+        files: ^dev-requirements\.(in|txt)$
+      - id: pip-compile
+        name: pip-compile requirements.in
+        args: [requirements.in, --upgrade, --generate-hashes, --allow-unsafe, --output-file, requirements.txt]
+        files: ^requirements\.(in|txt)$

Dockerfile

@@ -1,41 +1,50 @@
-FROM python:3.6-alpine3.8 AS builder
-
-# Install dependencies
-RUN apk add --update \
-    build-base \
-    g++ \
-    libffi-dev \
-    mariadb-dev \
-    jpeg-dev \
-    zlib-dev
+FROM hmsdbmitc/dbmisvc:debian11-slim-python3.10-0.5.0 AS builder
+
+# Install requirements
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        curl \
+        ca-certificates \
+        bzip2 \
+        gcc \
+        default-libmysqlclient-dev \
+        libssl-dev \
+    && rm -rf /var/lib/apt/lists/*
 
 # Add requirements
-ADD app/requirements.txt /requirements.txt
+ADD requirements.* /
 
-# Install Python packages
-RUN pip install -r /requirements.txt
+# Build Python wheels with hash checking
+RUN pip install -U wheel \
+    && pip wheel -r /requirements.txt \
+        --wheel-dir=/root/wheels
 
-FROM hmsdbmitc/dbmisvc:3.6-alpine
+FROM hmsdbmitc/dbmisvc:debian11-slim-python3.10-0.5.0
 
-RUN apk add --no-cache --update \
-    bash \
-    nginx \
-    curl \
-    openssl \
-    jq \
-    mariadb-connector-c \
-    jpeg-dev \
-    zlib-dev \
-  && rm -rf /var/cache/apk/*
+# Copy Python wheels from builder
+COPY --from=builder /root/wheels /root/wheels
 
-# Copy pip packages from builder
-COPY --from=builder /root/.cache /root/.cache
+# Install requirements
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        default-libmysqlclient-dev \
+        libmagic1 \
+    && rm -rf /var/lib/apt/lists/*
 
-# Add requirements
-ADD app/requirements.txt /requirements.txt
+# Add requirements files
+ADD requirements.* /
 
-# Install Python packages
-RUN pip install -r /requirements.txt
+# Install Python packages from wheels
+RUN pip install --no-index \
+        --find-links=/root/wheels \
+        --force-reinstall \
+        # Use requirements without hashes to allow using wheels.
+        # For some reason the hashes of the wheels change between stages
+        # and Pip errors out on the mismatches.
+        -r /requirements.in
+
+# Setup entry scripts
+ADD docker-entrypoint-init.d/* /docker-entrypoint-init.d/
 
 # Copy app source
 COPY /app /app
@@ -70,4 +79,8 @@ ENV DBMI_APP_STATIC_ROOT=/app/assets
 # Healthchecks
 ENV DBMI_HEALTHCHECK=true
 ENV DBMI_HEALTHCHECK_PATH=/healthcheck
-ENV DBMI_APP_HEALTHCHECK_PATH=/healthcheck
+ENV DBMI_APP_HEALTHCHECK_PATH=/healthcheck
+
+# File proxy
+ENV DBMI_FILE_PROXY=true
+ENV DBMI_FILE_PROXY_PATH=/proxy

app/contact/urls.py

@@ -1,8 +1,8 @@
-from django.conf.urls import url
+from django.urls import re_path
 from .views import contact_form
 
 app_name = 'contact'
 urlpatterns = (
-    url(r'^(?P<project_key>[^/]+)/?$', contact_form, name='contact_form'),
-    url(r'^', contact_form, name='contact_form'),
+    re_path(r'^(?P<project_key>[^/]+)/?$', contact_form, name='contact_form'),
+    re_path(r'^', contact_form, name='contact_form'),
 )

app/contact/views.py

@@ -1,10 +1,11 @@
 import logging
 
-from pyauth0jwt.auth0authenticate import public_user_auth_and_jwt
+from hypatio.auth0authenticate import public_user_auth_and_jwt
 
 from contact.forms import ContactForm
 
 from projects.models import DataProject
+from manage.views import is_ajax
 
 from django.http import HttpResponse, HttpResponseRedirect
 from django.urls import reverse
@@ -57,24 +58,30 @@ def contact_form(request, project_key=None):
                                  extra=context)
 
             # Check how the request was made.
-            if request.is_ajax():
+            if is_ajax(request):
                 return HttpResponse('SUCCESS', status=200) if success else HttpResponse('ERROR', status=500)
             else:
                 if success:
                     # Set a message.
                     messages.success(request, 'Thanks, your message has been submitted!')
                 else:
                     messages.error(request, 'An unexpected error occurred, please try again')
-                return HttpResponseRedirect(reverse('dashboard:dashboard'))
+                return HttpResponseRedirect(reverse(
+                    'projects:view-project',
+                    kwargs={'project_key': form.cleaned_data['project']}
+                ))
         else:
             logger.error("[HYPATIO][ERROR][contact_form] Form is invalid! - " + str(request.user.id))
 
             # Check how the request was made.
-            if request.is_ajax():
+            if is_ajax(request):
                 return HttpResponse('INVALID', status=500)
             else:
                 messages.error(request, 'An unexpected error occurred, please try again')
-                return HttpResponseRedirect(reverse('dashboard:dashboard'))
+                return HttpResponseRedirect(reverse(
+                    'projects:view-project',
+                    kwargs={'project_key': form.cleaned_data['project']}
+                ))
 
     # If a GET (or any other method) we'll create a blank form.
     initial = {}
@@ -114,7 +121,9 @@ def email_send(subject=None, recipients=None, email_template=None, extra=None):
         msg.attach_alternative(msg_html, "text/html")
         msg.send()
     except Exception as ex:
-        print(ex)
+        logger.exception(ex, exc_info=True, extra={
+            'email': email_template, 'extra': extra
+        })
         sent_without_error = False
 
     logger.debug("[HYPATIO][DEBUG][email_send] E-Mail Status - " + str(sent_without_error))