Please read the troubleshooting document which provides answers to previously asked questions and possible solutions to issues that have been reported.
-
The document can be found on GitHub or downloaded as a PDF.
Read through the sections in the technical overview relevant to your issue and check the topics below for possible solutions.
-
Does the issue persist and / or the provided information not help?
-
If you're a data producer , using the DDS for data deliveries to your users: contact the
+
Does the issue persist and/or the provided information not help?
+
If you're a data producer, using the DDS for data deliveries to your users: contact the
-
If you're a data recipient , using the DDS to download your data from a data producing SciLifeLab unit: contact the SciLifeLab unit responsible for delivering the data to you.
+
If you're a data recipient, using the DDS to download your data from a data producing SciLifeLab unit: contact the SciLifeLab unit responsible for delivering the data to you.
Please provide the following information:
-
Did the error / issue occurr in the web interface or the CLI?
+
Did the error/issue occur in the web interface or the CLI?
Web:
What did you attempt to do?
-
When did this message / error appear?
+
When did this message/error appear?
Screenshot of the web page with the error.
CLI
diff --git a/dds_web/version.py b/dds_web/version.py
index c0b18f39f..91ebb8d13 100644
--- a/dds_web/version.py
+++ b/dds_web/version.py
@@ -1,3 +1,3 @@
# Do not do major version upgrade during 2024.
# If mid or minor version reaches 9, continue with 10, 11 etc etc.
-__version__ = "2.7.1"
+__version__ = "2.8.0"
diff --git a/doc/Technical-Overview.pdf b/doc/Technical-Overview.pdf
deleted file mode 100644
index 1d0c3e67f..000000000
Binary files a/doc/Technical-Overview.pdf and /dev/null differ
diff --git a/doc/Troubleshooting.pdf b/doc/Troubleshooting.pdf
deleted file mode 100644
index c07302025..000000000
Binary files a/doc/Troubleshooting.pdf and /dev/null differ
diff --git a/doc/img/DDS-Overview.pdf b/doc/img/DDS-Overview.pdf
deleted file mode 100644
index 5b9a52f4e..000000000
Binary files a/doc/img/DDS-Overview.pdf and /dev/null differ
diff --git a/doc/img/Project-Statuses.pdf b/doc/img/Project-Statuses.pdf
deleted file mode 100644
index 4076edde2..000000000
Binary files a/doc/img/Project-Statuses.pdf and /dev/null differ
diff --git a/doc/img/User-Roles.pdf b/doc/img/User-Roles.pdf
deleted file mode 100644
index 00a4526a3..000000000
Binary files a/doc/img/User-Roles.pdf and /dev/null differ
diff --git a/doc/img/dds_overview.svg b/doc/img/dds_overview.svg
new file mode 100644
index 000000000..4ae991ccc
--- /dev/null
+++ b/doc/img/dds_overview.svg
@@ -0,0 +1,4 @@
+
+
+
+
\ No newline at end of file
diff --git a/doc/img/dds_project_statuses.svg b/doc/img/dds_project_statuses.svg
new file mode 100644
index 000000000..4aa239b06
--- /dev/null
+++ b/doc/img/dds_project_statuses.svg
@@ -0,0 +1,4 @@
+
+
+
+
\ No newline at end of file
diff --git a/doc/img/dds_raw_images.drawio b/doc/img/dds_raw_images.drawio
new file mode 100644
index 000000000..c16c447a7
--- /dev/null
+++ b/doc/img/dds_raw_images.drawio
@@ -0,0 +1,1826 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/doc/img/dds_roles.svg b/doc/img/dds_roles.svg
new file mode 100644
index 000000000..df946af55
--- /dev/null
+++ b/doc/img/dds_roles.svg
@@ -0,0 +1,4 @@
+
+
+
+
\ No newline at end of file
diff --git a/doc/img/en_and_decryption.svg b/doc/img/en_and_decryption.svg
new file mode 100644
index 000000000..72b316382
--- /dev/null
+++ b/doc/img/en_and_decryption.svg
@@ -0,0 +1,4 @@
+
+
+
+
\ No newline at end of file
diff --git a/doc/img/flowcharts/auth_login.pdf b/doc/img/flowcharts/auth_login.pdf
deleted file mode 100644
index 6b8cb6b52..000000000
Binary files a/doc/img/flowcharts/auth_login.pdf and /dev/null differ
diff --git a/doc/img/flowcharts/auth_twofactor_configure.pdf b/doc/img/flowcharts/auth_twofactor_configure.pdf
deleted file mode 100644
index e02af5758..000000000
Binary files a/doc/img/flowcharts/auth_twofactor_configure.pdf and /dev/null differ
diff --git a/doc/img/flowcharts/data_get.pdf b/doc/img/flowcharts/data_get.pdf
deleted file mode 100644
index dca62416f..000000000
Binary files a/doc/img/flowcharts/data_get.pdf and /dev/null differ
diff --git a/doc/img/flowcharts/data_put.pdf b/doc/img/flowcharts/data_put.pdf
deleted file mode 100644
index ed4dca1ec..000000000
Binary files a/doc/img/flowcharts/data_put.pdf and /dev/null differ
diff --git a/doc/img/flowcharts/project_create.pdf b/doc/img/flowcharts/project_create.pdf
deleted file mode 100644
index 8b94a5e72..000000000
Binary files a/doc/img/flowcharts/project_create.pdf and /dev/null differ
diff --git a/doc/img/flowcharts/user_add.pdf b/doc/img/flowcharts/user_add.pdf
deleted file mode 100644
index 2a4afb16b..000000000
Binary files a/doc/img/flowcharts/user_add.pdf and /dev/null differ
diff --git a/doc/img/flowcharts/user_add_project.pdf b/doc/img/flowcharts/user_add_project.pdf
deleted file mode 100644
index 858fa6610..000000000
Binary files a/doc/img/flowcharts/user_add_project.pdf and /dev/null differ
diff --git a/doc/img/flowcharts_2FA_setup.svg b/doc/img/flowcharts_2FA_setup.svg
new file mode 100644
index 000000000..1b5e2b301
--- /dev/null
+++ b/doc/img/flowcharts_2FA_setup.svg
@@ -0,0 +1,4 @@
+
+
+
+
\ No newline at end of file
diff --git a/doc/img/flowcharts_authentication.svg b/doc/img/flowcharts_authentication.svg
new file mode 100644
index 000000000..3578b87b6
--- /dev/null
+++ b/doc/img/flowcharts_authentication.svg
@@ -0,0 +1,4 @@
+
+
+
+
\ No newline at end of file
diff --git a/doc/img/flowcharts_create_project.svg b/doc/img/flowcharts_create_project.svg
new file mode 100644
index 000000000..fdacfec82
--- /dev/null
+++ b/doc/img/flowcharts_create_project.svg
@@ -0,0 +1,4 @@
+
+
+
+
\ No newline at end of file
diff --git a/doc/img/flowcharts_download.svg b/doc/img/flowcharts_download.svg
new file mode 100644
index 000000000..243f82151
--- /dev/null
+++ b/doc/img/flowcharts_download.svg
@@ -0,0 +1,4 @@
+
+
+
+
\ No newline at end of file
diff --git a/doc/img/flowcharts_invite_to_project.svg b/doc/img/flowcharts_invite_to_project.svg
new file mode 100644
index 000000000..bb21aa642
--- /dev/null
+++ b/doc/img/flowcharts_invite_to_project.svg
@@ -0,0 +1,4 @@
+
+
+
+
\ No newline at end of file
diff --git a/doc/img/flowcharts_inviting_users.svg b/doc/img/flowcharts_inviting_users.svg
new file mode 100644
index 000000000..86d534df8
--- /dev/null
+++ b/doc/img/flowcharts_inviting_users.svg
@@ -0,0 +1,4 @@
+
+
+
+
\ No newline at end of file
diff --git a/doc/img/flowcharts_upload.svg b/doc/img/flowcharts_upload.svg
new file mode 100644
index 000000000..340a60114
--- /dev/null
+++ b/doc/img/flowcharts_upload.svg
@@ -0,0 +1,4 @@
+
+
+
+
\ No newline at end of file
diff --git a/doc/old/Data-Delivery-System_SciLifeLab-Data-Centre_2020-11-20.pdf b/doc/old/Data-Delivery-System_SciLifeLab-Data-Centre_2020-11-20.pdf
deleted file mode 100644
index 006c299f4..000000000
Binary files a/doc/old/Data-Delivery-System_SciLifeLab-Data-Centre_2020-11-20.pdf and /dev/null differ
diff --git a/doc/technical-overview.md b/doc/technical-overview.md
new file mode 100644
index 000000000..d37040fb7
--- /dev/null
+++ b/doc/technical-overview.md
@@ -0,0 +1,1012 @@
+---
+title: "Data Delivery System"
+subtitle: "Technical Overview"
+output:
+ pdf_document:
+ md_extensions: +grid_tables
+toc: true
+geometry: "left=1cm, right=1cm, top=2cm, bottom=2cm"
+fontsize: 12pt
+colorlinks: true
+header-includes:
+ # Next 2 lines set the font to Lato
+ - \usepackage{lato}
+ - \renewcommand{\familydefault}{\sfdefault}
+ # Next 3 lines set the header to include the date
+ - \usepackage{fancyhdr}
+ - \pagestyle{fancy}
+ - \rhead{15-Jul-2024}
+ # Next 4 lines create cmd to rotate pages horizontally
+ - |
+ ```{=latex}
+ \usepackage{pdflscape}
+ \newcommand{\blandscape}{\begin{landscape}}
+ \newcommand{\elandscape}{\end{landscape}}
+ ```
+---
+
+\pagebreak
+
+## Abbreviations and terminology
+
+2FA
+: Two Factor Authentication
+
+API
+: Application Programming Interface
+
+Archiving
+: Changing the project status to Archive
+
+Authentication token
+: In the context of DDS, a file created upon successful user authentication and used for
+secure API calls during limited period of time
+
+CLI
+: Command Line Interface
+
+DDS
+: Data Delivery System
+
+DiA
+: Days in Available
+
+DiE
+: Days in Expired
+
+Kubernetes cluster
+: A set of node machines for running containerized applications
+
+MOTD
+: Message of the Day. Important message from DDS Super Admins to all users.
+
+NGI
+: SciLifeLab National Genomics Infrastructure
+
+PI
+: Principal Investigator
+
+PO
+: DDS Product Owner. Member of the SciLifeLab Data Centre responsible for the DDS project, including development and
+external contact such as with SciLifeLab units.
+
+Releasing
+: Changing the project status from In Progress to Available
+
+REST
+: REpresentational State Transfer[^1]
+
+Retracting
+: Changing the project status from Available to In Progress
+
+Safespring
+: Swedish provider of locally based Cloud services
+
+SUNET
+: Swedish University Computer Network
+
+Sysadmin
+: System administrator
+
+Unit
+: Within DDS, the representation of a SciLifeLab platform or one of the units within a platform
+
+Web (part)
+: Interface for accessing the DDS through the Internet using a web browser
+
+\pagebreak
+
+## 1. What is the Data Delivery System?
+
+The Data Delivery System (DDS) is a tool built for the simple and secure delivery of data from
+SciLifeLab platforms to their users. The system uses Safespring’s[^2] object storage service as the
+delivery medium, thereby keeping the data within the Swedish borders. The DDS has built-in
+encryption and key management, enabling protection of the data without requiring the users to
+perform these steps themselves prior to delivery.
+
+
+
+The DDS has been developed by the SciLifeLab Data Centre together with NGI Stockholm. The web
+interface can be found at and the documentation at
+[GitHub pages](https://scilifelabdatacentre.github.io/dds_cli/).
+
+### What is it not?
+
+The system is not a storage solution; The data will only be stored for a short period of time[^3] and the
+recipients are themselves responsible for the future handling of their data. Backup of the data is not
+provided during the data delivery. The system is also not built for data sharing; it is purely for delivery
+of data from one data producer to a data owner.
+
+## 2. How is it used?
+
+The DDS is hosted on a Kubernetes cluster[^4] owned and maintained by the SciLifeLab Data Centre.
+The system consists of two main components: a web part (“web”) and a REST API (“API”). The vast
+majority of the DDS functionality resides in the API, which can be utilized via the DDS command line
+interface (CLI). At this time, the web can be used for user registration ([Registration](#registration)), changing and
+resetting passwords and listing the projects. In addition, the web links to the code base,
+documentation, troubleshooting information, this technical overview and the Python Package Index
+([PyPi](https://pypi.org/project/dds-cli/)). In order to meet the needs of more users (e.g. those not as familiar with
+the command line) we plan to include more functionality currently present in the CLI into the web.
+This section describes the standard DDS use-case and the flow of data within the system. The steps
+do not include actions taken prior to data delivery, such as sample collection, the production of the
+data or the communication between customer (user) and SciLifeLab platform.
+Accounts within the system will be referred to by the role of the account in question.
+
+- A Super Admin refers to the person behind a DDS account with the role “Super Admin”.
+- A Unit Admin refers to the person behind a DDS account with the role “Unit Admin”. Unit
+ Admins are associated with a SciLifeLab unit.
+- A Unit Personnel refers to the person behind a DDS account with the role “Unit Personnel”.
+ Unit Personnel are associated with a SciLifeLab unit.
+- A Researcher refers to the person behind a DDS account with the role “Researcher”. A
+ Researcher is any person who uses the services of a SciLifeLab unit.
+- A Project Owner refers to the person behind a DDS account with the role “Researcher”, but
+ who is marked as the owner of a specific project. For more information on the account roles
+ and their permissions within the DDS, please see the appendix ([User Roles](#a-user-roles)).
+
+### 2.1. Delivery Overview
+
+Sections 2.2 to 2.11 provide a more detailed description of the following steps.
+
+1. A Super Admin creates a unit in the DDS
+2. A Super Admin invites Unit Admins
+3. The Unit Admins register their accounts in the DDS
+4. The Unit Admins invite Unit Personnel
+5. The Unit Personnel register their accounts in the DDS
+6. A Unit Admin/Personnel creates a project
+7. A Unit Admin/Personnel uploads data
+8. A Unit Admin/Personnel releases the data to the recipient
+9. A Researcher/Project Owner downloads data
+10. The project status is automatically changed to expired after a number of days
+11. If the project status has not been changed to available again within a number of days, the
+ project is automatically archived
+
+### 2.2. Creating a Unit in the DDS
+
+In order to begin using the DDS, a SciLifeLab platform or unit must contact the SciLifeLab Data
+Centre at [delivery@scilifelab.se](mailto:delivery@scilifelab.se). When the necessary agreements are in place, the Data
+Centre should have the following information; The information is required in order to add the unit to the
+production instance, and should be collected from the agreements or additional contact with the unit.
+
+| Parameter | Definition |
+|---------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Unit Name* | The name of the unit or platform. |
+| External Unit Name | The unit name to display to users e.g. in emails. If not specified by the unit, the Data Centre will set this to the same value as Unit Name. |
+| Contact Email* | The email address on which the unit would like to be contacted regarding the DDS. This address will be displayed in some emails sent by the DDS and on the web page, e.g. when renewal of project access is required. |
+| Public ID | An identifier for the SciLifeLab unit (e.g. abbreviation of the SciLifeLab unit name) which may be displayed publicly. If not chosen by the unit, the Data Centre will choose an appropriate abbreviation. Preferably, this ID should be a short abbreviation of the unit name, and should adhere to the following rules. Only contain letters, digits, dots (.) and hyphens (-). Begin with a letter or digit. Contain a maximum of two dots. Not start with “xn--”. |
+| Internal reference ID | At this time, this specifies the prefix of the public ID of the future projects within the unit, and should adhere to the same rules as the Public ID. If this ID is not set, the value will default to the Public ID. |
+| Days in Available | The number of days during which the data will be available for download by the Researchers. The countdown starts when the project is released. There is no time limit when the project is In Progress and the project has not been released. For more information on the project statuses and what actions can be performed during them, see the appendix ([Project Statuses](#b-project-statuses)). After Days in Available (DiA) number of days has passed, the project is automatically set as Expired. |
+| Days in Expired | The number of days (after being available) during which the data is still stored but not available for download. During this time, the project can be renewed, leading to the project being available again and therefore allowing for downloads again. When the Days in Expired (DiE) number of days has passed, the project is automatically archived by the system. |
+| Quota | The amount of storage space made available for a particular unit. This information should be included in the service agreement. Another value cannot be chosen by the Data Centre or by the unit. |
+| Warning level | When a unit has used this percentage of it’s available storage space, an alert is triggered and sent to [delivery@scilifelab.se](mailto:delivery@scilifelab.se). In the event of an alert, the Data Centre contacts the unit to discuss whether or not the quota is sufficient. If not, the service agreement will need to be updated and the quota increased in both the database and at Safespring's S3 storage, for that specific Safespring project. |
+: Parameters used to configure a Unit within DDS
+
+The unit also must provide the email addresses of at least two (but preferably three or more)
+individuals which should have the DDS account role Unit Admin. For more information on why this
+is required, see the section [Creating a Project](#creating-a-project).
+
+When this information has been received, the Data Centre contacts Safespring in order to create a
+new Safespring project (not to be confused with a delivery project). The Safespring project is the
+location within the cloud where the uploaded data will be stored. When a Safespring project is
+created, the DDS Product Owner (“PO”) receives the following information from Safespring:
+
+Keys
+: Access key and secret key. These are created within Safespring and can be retrieved from there by the Data Centre.
+
+Endpoint
+: The storage site. This is currently the same for all, but may change in the future, either for all or some of the
+Safespring projects.
+
+Name
+: A unique name for the Safespring project. This is currently not used for anything aside from keeping track but will
+be useful information once the invoicing system has been implemented.
+
+A system administrator (sysadmin) in the Data Centre accesses the DDS deployment and runs a
+command using the information described above. This inserts the unit information into the database.
+The database is encrypted with the Advanced Encryption Standard (AES) Counter (CTR) mode,
+with a 512 bit key.
+
+### 2.3. Registration
+
+In order to obtain an account, another user already having an account in the system needs to send
+an invite [^5]. When a unit is first created in the DDS, the Super Admin invites Unit Admins to the specific
+unit. When invited, the Unit Admin receives an email from the DDS with the subject “[USER] invites
+you to the SciLifeLab Data Delivery System“. The email contains a link to the registration page. When
+following the link, the user is prompted to enter the following information. All fields are required.
+
++-------------------+----------------------------------------------------------+----------------------------------------------------------------------------------+
+| Field | Requirement(s) | Note |
++===================+==========================================================+==================================================================================+
+| Name | At least two characters long | The full name of the person registering an account. |
++-------------------+----------------------------------------------------------+----------------------------------------------------------------------------------+
+| Username | Not taken by another user | It is not possible to change the username at this time. Please save the |
+| | Between three and 30 characters long | username in a secure way, e.g. a password management software. |
+| | Can only contain the following characters: | |
+| | | |
+| | - Letters | |
+| | - Digits | |
+| | - Underscore (_) | |
+| | - Dot (.) | |
+| | - Dash (-) | |
++-------------------+----------------------------------------------------------+----------------------------------------------------------------------------------+
+| Password | Between 10 and 64 characters long | Please remember your password. We highly recommend storing it using a |
+| | Must contain at least: | password management software such as BitWarden or LastPass. |
+| | | |
+| | - One upper case letter | Forgetting your password means potential loss of access to data. You can reset |
+| | - One lower case letter | your password, however this means that another user needs to renew your project |
+| | - One digit or special character | access if there are any active and ongoing projects. If your access is not |
+| | | renewed, or all users that are able to renew your access also have forgotten |
+| | | their passwords, any data that has been uploaded will not be possible to |
+| | | download or decrypt and therefore a new delivery project will be needed. |
+| | | |
+| | | Super Admins do not have access to any of the data or your passwords and |
+| | | therefore cannot help you retrieve any data if you lose project access. |
++-------------------+----------------------------------------------------------+----------------------------------------------------------------------------------+
+| Email Address | Not registered to another account. | This cannot be changed as the invite is specific to one email. In addition one |
+| | | email address can only be connected to one account and by extension one account |
+| | | role. |
++-------------------+----------------------------------------------------------+----------------------------------------------------------------------------------+
+: Required information to create any user in DDS
+
+Once a Unit Admin has registered an account, they can invite Unit Personnel. Unit Personnel can
+only be invited to the unit to which the Unit Admin is connected. Super Admins, Unit Admins and Unit
+Personnel can all invite Researchers. All invited users get identical emails as described above, and
+register their account in the DDS by following the link and filling in the same required information.
+
+### 2.4. Authentication
+
+Before delivering data with the CLI, it needs to be installed (see the
+[Installation Guide](https://scilifelabdatacentre.github.io/dds_cli/installation/) located in
+GitHub pages) and all users are required to authenticate themselves. The authentication process
+via CLI is as follows.
+
+1. The user runs [^6] the authentication command and the CLI asks for the username and
+ password.
+2. Assuming the username and password are correct, the user is asked to authenticate with
+ two factor authentication (2FA). The first authentication requires 2FA via email: An email
+ containing an eight digit code is sent to the user's registered email and the user is prompted
+ to enter the code. After this, the user can set up 2FA via authenticator app [^7] if they wish. If
+ not, the default method of 2FA via email will be used for all authentication within the DDS.
+3. Assuming that the 2FA succeeds, an encrypted token is saved on the local machine or server,
+ depending on where the command is executed. This token acts as an authenticated session
+ and will be used in the succeeding commands. The default destination of the encrypted
+ authenticated token is the home directory. If a specific destination is specified in the
+ authentication command, the token will be saved in that location [^8]. The token is valid for seven
+ days. We highly recommend you to re-authenticate before uploading or downloading any
+ data [^9].
+ The same procedure is followed when logging into the web; The username and password is entered,
+ followed by 2FA with the chosen 2FA method. The web session however, in contrast to the CLI
+ session, only lasts for one hour. If a user is inactive for more than one hour, they will be asked to log
+ in again. As a final note on the authentication process, a user can attempt the authentication via CLI
+ and login via the web a maximum of ten times in one hour.
+
+### 2.5. Inviting users
+
+The CLI is used to invite any type of user[^10]. The following list describes which roles can invite which
+roles, if any, and if there are any exceptions or additional important information to note.
+
++----------------+------------------+----------------+------------------+-----------------------+
+| Inviter role | Invitee role | | | |
++----------------+------------------+----------------+------------------+-----------------------+
+| | Super Admin | Unit Admin | Unit Personnel | Researcher |
++================+==================+================+==================+=======================+
+| Super Admin | A Super Admin can invite any type of user to the DDS in general. They |
+| | cannot, however, invite users to specific projects, since they do not |
+| | have access to any projects themselves or any of the data uploaded |
+| | within them. |
++----------------+------------------+----------------+------------------+-----------------------+
+| Unit Admin | | Unit Admins can invite Unit Admins|Unit Admins/Personnel |
+| | | /Personnel to the unit which their|can invite Researchers,|
+| | x | account is associated with. [^11] |both within the system |
+| | | |and to specific |
+| | | |projects associated |
+| | | |the inviter's unit |
++----------------+------------------+----------------+------------------+-----------------------+
+| Unit Personnel | | |Unit Personnel |Unit Admins/Personnel |
+| | | |can invite other |can invite Researchers,|
+| | x | x |Unit Personnel to |in general, and to |
+| | | |the unit which |specific projects |
+| | | |their account is |associated with the |
+| | | |associated with. |inviter's unit |
++----------------+------------------+----------------+------------------+-----------------------+
+| Researcher | | | |Researchers cannot in |
+| | | | |general invite any |
+| | | | |other users. However, |
+| | | | |if the Researcher is |
+| | | | |marked as the owner |
+| | x | x | x |(Project Owner) of a |
+| | | | |specific project, they |
+| | | | |can invite both other |
+| | | | |Project Owners and |
+| | | | |Researchers into that |
+| | | | |specific project. |
++----------------+------------------+----------------+------------------+-----------------------+
+: Privileges of different users within the system
+
+### 2.6. Creating a Project
+
+Unless otherwise stated, a project refers to a delivery project - a container in the DDS to which a unit
+uploads one or more data files to be delivered to a researcher.
+
+Both Unit Admins and Unit Personnel can create delivery projects for their unit. However, a unit will
+not be able to create any delivery projects until there are at least two accounts with the role “Unit
+Admin” for the specific unit. The reasoning behind this is explained below:
+
+---
+
+- When a user registers an account in the DDS ([Registration](#registration)), a key pair is generated
+ (“user key pair”) for that specific user. The public part of this pair (“user public
+ key”) is stored in the database without any protection apart from the encryption
+ affecting the entire database. The private part of the key pair (“user private key”),
+ however, is encrypted by a key derived from the password chosen during registration
+ and thereafter saved to the database.
+
+- A key pair is also generated when a project is created (“project key pair”). In the
+ same way as the user public key, the public part of the project key pair (“project
+ public key”) is saved to the database without an extra layer of encryption. The
+ private part (“project private key”) is encrypted with the user public key.
+
+- The above means that the project private key can only be decrypted with the user
+ private key, which can only be decrypted with a key derived from the user's password.
+ Since the project private key is required for decrypting all data within a specific
+ project, forgetting the DDS password and resetting it also results in the loss of
+ data access. Note that this is not the case for changing the password, where the
+ user logs in to the web and provides the old password in order to set a new one.
+ This does not affect the data access in any form.
+
+- The reasoning behind requiring at least two Unit Admins for a specific unit is
+ to reduce the risk that all Unit Admins associated with a specific unit lose access
+ to the project data at the same time. If this would occur, they would not be able to
+ fix the others access [^12] and Unit Personnel cannot fix the project access for Unit
+ Admins.
+
+- Note: Users only choose and keep track of their username and password. All other
+ keys are generated by the system and stored in the database.
+
+---
+
+If a unit has two Unit Admins, any user attempting to create a project will be prompted with a warning,
+informing them of the risk of data access loss [^13]. Once a unit has at least three Unit Admins this
+warning will disappear.
+
+The following information is required (and saved to the database) when creating a project:
+
+Title
+: The title of the project. There is no upper limit to the length of this,
+however it must be at least one character long and it can only contain
+letters, digits and spaces - special characters are not allowed. It is
+also not unique; there can be multiple projects with identical titles.
+
+Description
+: A description of the project. It needs to be at least one character long,
+but there is no upper limit. In addition to letters, digits and spaces,
+special characters are allowed in the description.
+
+Principal Investigator
+: An email address to the Principal Investigator (PI). Note that this
+email address is not used for anything other than information at this
+time. It is not connected to any DDS account, and they receive no
+emails. This is due to the fact that some PIs are not involved in the
+data collection, and may be listed as the PI as a formality. Thus, the
+PI does not necessarily have to have a DDS account. If the PI should
+receive emails and have a DDS account with access to your project,
+we suggest that you set the same person as the Project Owner. This
+can be done in two ways:
+
+1. Creating a project and then adding users to the project[^14].
+2. Creating a project and at the same time specifying which Researchers should have access and which user(s) should
+ be set as an owner[^15].
+
+In addition to this, the following information is generated for each project:
+
+Public ID
+: The public ID is generated from the Internal Reference ID ([Creating a Unit
+in the DDS](#creating-a-unit-in-the-dds)) and a counter. Thus, each project public ID associated with a
+specific unit will have the same prefix. For example, if the Internal
+Reference ID for the associated unit is intref, the first project created for
+that unit would be intref00001, the second intref00002, and so on.
+
+Bucket
+: The name of the project-specific bucket with the Safespring project. This
+is generated from the public ID, the current time (Date Created below)
+and a randomly generated string of characters.
+
+Date Created
+: A timestamp representing the date and time when the project was
+created.
+
+Date Updated
+: A timestamp representing the date and time when the project was last
+updated, e.g. a folder or file was deleted.
+
+Key Pair
+: As described in the beginning of this section, a key pair is generated for
+each project. This key pair consists of a public and a private part. The
+public part (“project public key”) is saved together with the project
+information and there is a single copy. The private part (“project private
+key”) is encrypted with the user public key of each user with access to the
+project in question. Thus, there is one copy of the project private key per user.
+
+Sensitive
+: Information on whether the project is regarded as sensitive or not. If any
+sensitive data, such as (for example) human related data, will be
+delivered at any stage in a delivery project, the project should be set as
+sensitive. This is the default. There is an option to specify that the project
+can be regarded as non-sensitive, however at this time the handling of
+sensitive and non-sensitive project data are identical: all data is encrypted
+and decrypted. This may change in a future version.
+
+Released
+: Information on whether or not the project data has been released (project
+status set to “Available”) at some point. For more information on the
+project statuses, please see the appendix ([Project Statuses](#b-project-statuses)).
+
+Active
+: Whether or not the project is currently active. The default is active. The
+following statuses are regarded as active: - In Progress - Available - Expired
+The following statuses are regarded as non-active: - Deleted - Archived (incl. aborted)
+
+### 2.7. Uploading data
+
+Once a project has been created, the project status is automatically set to In Progress. This status
+(and this status only) allows for upload of data within the project[^16]. Only Unit Admins and Unit
+Personnel have the permissions to upload data.
+When starting an upload, a directory (“staging directory”) is created by the executing command. The
+default location of the staging directory is the current working directory, however the user can specify
+an existing directory in which the staging directory should be placed. Independent of the location
+(specified or default), the staging directory is named _DataDelivery**_,
+where is the date and time when the upload was started, and is the ID of
+the project the user is attempting to upload data to. If there is no data to upload, this directory is
+deleted immediately. If not, the staging directory will contain three subdirectories:
+
+logs
+
+: The logs directory will be empty if the upload is successful. If an error occurs
+during the upload, a log file called `"dds_failed_delivery.json"` will be
+created and placed in the logs directory. If the actual upload to the cloud is
+successful but the file information (e.g. name, checksum, size etc.) fails to be
+saved to the database, the log file lets the PO insert the missing file data into the
+database. Without this information, the uploaded file(s) will be located in the
+cloud but will not be accessible by any of the user roles since there is no record
+of it/them in the database. Thus, in the case of this log file being generated,
+contact the Data Centre immediately and attach the log file to the message. If
+you do not contact us with this information, the DDS cleanup actions[^17] will:
+ - Remove the files found in the cloud which are missing database metadata
+ - Remove rows in the database which are missing the corresponding file in the cloud
+
+files
+: This is the subdirectory where the actual staging occurs. Prior to the upload of
+a file, the DDS checks whether or not the file appears to be compressed by
+examining the file signature contained in the first few bytes of the file. If the file
+appears to be compressed, no compression is performed on the file. If the file
+appears to be in raw format, it is compressed by the DDS[^18]. As mentioned
+previously, all files are also encrypted. When encrypting (and possibly
+compressing) the files, the new files (“staged files”) are placed in the files
+directory. As soon as the file has been uploaded, the staged file is deleted. The
+original file is not affected.
+
+meta
+: The meta directory will always be empty. This is a remnant of a previous version
+and has not yet been removed.
+The staging process and upload is executed with several concurrent threads. This means that the
+DDS does not handle one file at a time. Instead, the default is for the DDS to stage and upload four
+files at a given time. However, this may vary (e.g. may be less) depending on the computer or server
+capabilities, and can also be changed by using one of the CLI options. Assuming that the default is
+used and that more than four files are included in the ongoing upload, there will be a maximum of
+four progress bars displayed in the terminal. The file names are displayed to the left of the bars, and
+to the left of the file names is the current ongoing process for each file. A lock indicates that a file is
+being compressed and encrypted, and an arrow pointing up indicates that the file is being uploaded
+to the bucket.
+Uploading the same data (files or directories with the same names/paths) again will by default not
+succeed. However, the user can explicitly tell the DDS to upload the same data again - to overwrite
+the current file information in the database and to upload a new version of the files. This will generate
+a new file version in the database, which is kept track of for invoicing purposes.
+
+Uploading data to the cloud requires collection[^19] of the Safespring keys which provides the
+connection between the CLI and Safesprings cloud storage. The plan is to change this to use a
+similar method as what the download ([Downloading data](#downloading-data)) uses - pre-signed urls. The plan is also to
+allow for the continuation of a failed upload. Currently, a failed upload requires the restart of the
+process. For a detailed flowchart describing the upload process, see the [Flowcharts](#d-flowcharts).
+
+### 2.8. Releasing the data
+
+The project status In Progress does not allow for any data download by Researchers. In order to
+give the Researchers access to the data, the project needs to have the status Available[^20]. The
+process of changing the project status from In Progress to Available is hereby referred to as
+“releasing” the data/project. When the data is released, the countdown of the DiA[^21] ([Creating a
+Unit in the DDS](#creating-a-unit-in-the-dds)) starts, and upload is no longer possible. In the case that some file has been missed
+during the upload, the Unit Admin/Personnel can change the project status from Available to In
+Progress (“retracting”). This lets the Unit Admin/Personnel continue uploading files and prevents
+any downloads by the Researchers. However, retracting the project does not pause the countdown
+of DiA. When DiA number of days have passed, the project status is automatically set to Expired.
+The deadline for a given project can be extended by Unit Admins or Personnel.
+More information on this can be found in section [Automatic expiry of data access](#automatic-expiry-of-data-access-and-archiving-of-project).
+Releasing the data also, by default, notifies the Researchers that there is data available for
+download. If the unit does not want these emails to be sent, it can be disabled when performing the
+status change.
+
+### 2.9. Downloading data
+
+Once the project has been released, the Researchers with access to the project can begin
+downloading the data. Available is the only status that allows for data download by Researchers,
+and upload and deletion of any project contents by Unit Admin and Unit Personnel is not possible.
+When downloading data, the Researchers can either choose to download specific file(s), specific
+folder(s), or the entire project contents.
+
+As with the upload ([Uploading data](#uploading-data)), a staging directory is created when downloading the data. This
+directory is placed by default in the current working directory, and is named DataDelivery**.
+However unlike the upload command, downloading allows the user to choose the name of the directory - specifying a
+destination. The destination cannot be an existing directory[^22] - it must be a new directory. Since a new destination is required with every
+download, downloading the same file(s) multiple times is possible and is only limited by the amount
+of available storage space on the client (computer or server where the command is run).
+The staging directory contains the same subdirectories as the upload creates, however the usage
+of them differs slightly.
+
+logs
+: The logs directory will be empty if the download is successful. If an error occurs
+: The logs directory will be empty if the download is successful. If an error occurs
+during the download, a log file called `"dds_failed_delivery.json"` will be
+created and placed in the logs directory. In the case of this log file being
+generated, contact the Data Centre and attach the log file to the message.
+
+files
+: This is the subdirectory where the actual staging occurs. This is also the final
+destination of the downloaded and decrypted files. All downloaded files are
+saved to this location. Succeeding the download, all files are decrypted and
+those that were initially compressed by the DDS prior to upload are
+decompressed. As soon as a file has been decrypted and decompressed, the
+staged (downloaded) file is deleted from the files subdirectory. Downloading
+data does not affect any of the files in the cloud.
+
+meta
+: As during upload, the meta directory will always be empty.
+The download and decryption/decompression process is also executed with several concurrent
+threads. As for the upload, the download also has four as the default number of threads and this is
+adjustable in the same way. Assuming that the default is used and that more than four files are
+included in the ongoing download, there will be a maximum of four progress bars displayed in the
+terminal. The file names are displayed to the left of the bars, and to the left of the file names is the
+current ongoing process for each file. An arrow pointing up indicates that the file is being downloaded
+from the bucket, and a lock indicates that a file is being decrypted and decompressed.
+In contrast to the upload, the download of data from the cloud does not require the collection of the
+Safespring keys. Instead, the download is handled with the use of pre-signed urls. These urls are
+generated and cryptographically signed within the DDS when a user is attempting to download the
+data. If the download of a file fails, the process must start from scratch again. For a detailed flowchart
+describing the download process, see the [Flowcharts](#d-flowcharts).
+
+### 2.10. Automatic expiry of data access and archiving of project
+
+After DiA number of days, the DDS automatically changes the project status to Expired. This starts
+the countdown of the DiE number of days. In this status, all data and metadata is kept in the system,
+but all uploads, downloads and deletions of data by all roles are blocked. After DiE days, the project
+status is automatically changed to Archived. This is the end of a project life cycle[^23] and all files are
+deleted.
+If Researchers associated with the project need access to download the data, they must contact the
+responsible SciLifeLab unit. A Unit Admin or Unit Personnel within the DDS can then change the
+project status to Available again, thereby allowing for download and resetting the countdown of the
+DiA. Similarly, if additional data should be delivered in the project, a Unit Admin or Unit Personnel
+must first change the project status to Available and from there to In Progress[^24].
+The renewal of data access can be performed a maximum of two times. Once a project expires, it
+can be released again, then expire again and then released one last time. After this, the project
+expires, and can then only be archived.
+
+# Appendix
+
+## A User Roles
+
+There are four different roles in the DDS: “Super Admin”, “Unit Admin”, “Unit Personnel” and
+“Researcher”. The figure below shows a simplification of the permission hierarchy of these roles:
+
+- Researchers can list projects and their contents and download data, but only have
+ administrative permissions if they are set as an owner of a specific project.
+- Unit Personnel can list projects and their contents, download data, upload data and delete it.
+ They also have administrative permissions regarding other Unit Personnel and Researchers.
+- Unit Admins can list projects, their contents, download data, upload data and delete data.
+ They have administrative permissions regarding other Unit Admins and also Unit Personnel
+ and Researchers.
+- Super Admins do not have access to any data but have administrative permissions regarding
+ all other roles, including other Super Admins.
+
+
+
+### Super Admin
+
+The Super Admin role is reserved for certain employees of the SciLifeLab Data Centre.
+The table below shows which permissions a Super Admin has (“Yes”) and does not have (“No”). \* indicates
+that the permissions relate to all roles. If a role has a permission relating to one or more specific roles, these
+roles are listed below the permission.
+
++--------------------+------------------------------------------------------------------+-----------------------------------------------------------------+
+| Type of permission | Yes | No |
++====================+==================================================================+=================================================================+
+| Delivery related | - List all projects | - Create projects |
+| | *Note, however, that a Super Admin does not have access | - List project contents |
+| | to the projects and can therefore not access any data | - Download data |
+| | uploaded within it. Also, when the functionality to display | - Upload data |
+| | detailed information about individual projects has been | - Delete data |
+| | implemented, Super Admins will not be able to use that | - Change project statuses |
+| | functionality. The Super Admins can only see the information | |
+| | displayed with the project listing functionality. At this time,| |
+| | this is for the purpose of assisting with user queries. It will| |
+| | be investigated and possibly changed.* | |
++--------------------+------------------------------------------------------------------+-----------------------------------------------------------------+
+| Administrative | - Create units | |
+| | - Create MOTD’s | |
+| | - List units | |
+| | - List users | |
+| | - Invite users * | |
+| | - List invited users * | |
+| | - Activate and deactivate users * | |
+| | - Delete users * | |
+| | - Delete invitations * | |
+| | - Reset 2FA for users * | |
++--------------------+------------------------------------------------------------------+-----------------------------------------------------------------+
+: Permissions of Super Admins
+
+### Unit Admin
+
+The table below shows which permissions a Unit Admin has (“Yes”) and does not have (“No”). \* indicates that
+the permissions relate to all roles. If a role has a permission relating to one or more specific roles, these roles
+are listed below the permission.
+
++--------------------+--------------------------------------------------+---------------------------------------------+
+| Type of permission | Yes | No |
++====================+==================================================+=============================================+
+| Delivery related | - Create projects | - Any actions pertaining to |
+| | - List projects | *Super Admins* |
+| | - List project contents | - Activate and deactivate |
+| | - Upload data | *Researchers* |
+| | *Only when project status is In Progress.* | *This is due to the fact that |
+| | - Download data | Researchers can be involved in the |
+| | - Delete data | projects of multiple different* |
+| | *Only when project status is In Progress.* | *units. If a Unit Admin would like |
+| | | to deactivate a Researcher, please |
+| | | contact the PO (also *Super Admin).* |
+| | | - Delete *Researchers* |
+| | | *Same reasoning as for activating |
+| | | and deactivating Researchers, as |
+| | | described above.* |
+| | | - Reset 2FA for users * |
++--------------------+--------------------------------------------------+---------------------------------------------+
+| Administrative | - Invite users: | |
+| | - Unit Admins | |
+| | - Unit Personnel | |
+| | - Researchers | |
+| | - Add Researchers to specific | |
+| | projects, including Project Owners | |
+| | - Grant, revoke and fix users access to | |
+| | projects when access has been lost | |
+| | - Unit Admins | |
+| | - Unit Personnel | |
+| | - Researchers | |
+| | - List users | |
+| | *Only other Unit Admins and Unit Personnel | |
+| | associated to the same unit, and Researchers | |
+| | with access to the unit's projects.* | |
+| | - List invitations | |
+| | *Only invites of Unit Admins and Unit | |
+| | Personnel associated to the same unit, | |
+| | and Researchers invited to projects | |
+| | associated to the unit.* | |
+| | - Activate and deactivate users: | |
+| | - Unit Admins | |
+| | - Unit Personnel | |
+| | - Delete users: | |
+| | - Unit Admins | |
+| | - Unit Personnel | |
+| | - Delete invites | |
+| | - Unit Admins | |
+| | - Unit Personnel | |
+| | - Researchers | |
++--------------------+--------------------------------------------------+---------------------------------------------+
+: Permissions of Unit Admins
+
+### Unit Personnel
+
+The table below shows which permissions a Unit Personnel has (“Yes”) and does not have (“No”). \* indicates
+that the permissions relate to all roles. If a role has a permission relating to one or more specific roles, these
+roles are listed below the permission.
+
++--------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------+
+| Type of permission | Yes | No |
++====================+=====================================================================================================================================================================+=================================================================================================+
+| Delivery related | - Create projects | |
+| | - List projects | |
+| | - List project contents | |
+| | - Upload data | |
+| | *Only when project status is In Progress.* | |
+| | - Download data | |
+| | - Delete data | |
+| | *Only when project status is In Progress.* | |
++--------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------+
+| Administrative | - Invite users: | - Any actions pertaining to *Super Admins* and *Unit Admins* |
+| | o Unit Personnel | - Activate and deactivate users |
+| | o Researchers | - Delete users * |
+| | - Add Researchers to specific projects, including Project Owners | - Delete invited users * |
+| | - Grant, revoke and fix users access to projects when access has been lost | - Reset 2FA for users * |
+| | o Unit Personnel | |
+| | o Researchers | |
+| | - List users | |
+| | *Only other Unit Admins and Unit Personnel associated to the same unit, and Researchers with access to the unit's projects.* | |
+| | - List invitations | |
+| | *Only invites of Unit Admins and Unit Personnel associated to the same unit, and Researchers invited to projects associated to the unit.* | |
++--------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------+
+: Permissions of Unit Personnel
+
+### Researcher
+
+The Researcher role represents the data recipient. Project Owner is a subcategory of the Researcher
+role, and is intended for those Researchers that should have some administrative permissions within
+specific projects. Thus, the Project Owner is not a DDS user role, but a “role” within a certain project.
+The first table below reflects the permissions of Researchers in general. The second table reflects
+the permissions of Researches which have been marked as Project Owners within a specific project.
+
+The table below shows which permissions a Researcher has (“Yes”) and does not have (“No”). \* indicates
+that the permissions relate to all roles. If a role has a permission relating to one or more specific roles, these
+roles are listed below the permission.
+
++--------------------+-------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------+
+| Type of permission | Yes | No |
++====================+=============================================================================================================+===============================================================+
+| Delivery related | - List projects | - Create projects |
+| | Not all projects, only those the user has been associated with | - Upload data |
+| | by a Unit Admin, Unit Personnel or Project Owner. | - Delete data |
+| | - List project contents (uploaded data) | - Reset 2FA for users * |
+| | Only when project status is Available | |
+| | - Download data | |
+| | Only when project status is Available | |
++--------------------+-------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------+
+| Administrative | Researchers have no administrative permissions. |
++--------------------+-------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------+
+: Permissions of Researchers
+
+#### Project Owner
+
+The table below shows which permissions a Project Owner has (“Yes”) and does not have (“No”) in a
+specific project. \* indicates that the permissions relate to all roles. If a role has a permission relating to one
+or more specific roles, these roles are listed below the permission.
+
++--------------------+-------------------------------------------------------------------+--------------------------------------------------------------+
+| Type of permission | Yes | No |
++====================+===================================================================+==============================================================+
+| Delivery related | - List project contents (uploaded data) | - Create projects |
+| | *Only when project status is Available* | - Upload data |
+| | - Download data | - Delete data |
+| | *Only when project status is Available.* | |
++--------------------+-------------------------------------------------------------------+--------------------------------------------------------------+
+| Administrative | - Invite and add Researchers (including other Project Owners) | - Any actions pertaining to *Super Admins, Unit Admins and |
+| | to project | Unit Personnel* |
+| | - Grant, revoke and fix Researchers (including other Project | - Delete users * |
+| | Owners) access to projects when access has been lost | - Delete invited users * |
+| | | - Activate and deactivate users * |
+| | | - Add or invite users to other projects |
+| | | - Reset 2FA for users * |
++--------------------+-------------------------------------------------------------------+--------------------------------------------------------------+
+| | - List invites | |
+| | *Only invites of Researchers, where invite is to own projects.* | |
++--------------------+-------------------------------------------------------------------+--------------------------------------------------------------+
+: Permissions of Project Owners
+
+## B Project Statuses
+
+There are five different possible project statuses available within the DDS. The section [“How is it
+used?”](#how-is-it-used) describes the standard project life cycle, from creating a project with the automatic status In
+Progress, to project archivation. The figure below shows a summary of the different statuses and
+what they mean. The rest of this section describes them in more detail.
+
+
+
+### In Progress
+
+A project is automatically set to In Progress when it’s created. This status is to allow upload by Unit
+Admins and Unit Personnel and indicate that the data delivery is in progress. If invited to a project
+during this stage, Researchers can also see the project in their list of projects, however they cannot
+see if anything has been uploaded.
+The permissions of Unit Admins and Unit Personnel vary depending on whether or not the project
+has just been created (A. First time as In Progress), or if it has been released and later retracted one
+or more times (B. Nth time as In Progress).
+
+#### A. First time as In Progress
+
+The first time as In Progress, data can be uploaded, downloaded, deleted and overwritten. The
+project can also be Deleted, indicating that something has gone wrong before the data was released
+to the recipients.
+
+#### B. Nth time as In Progress
+
+If the project has been made Available and then retracted to In Progress again, deleting the project
+is no longer an option. In this case, the Unit Admin or Unit Personnel will need to archive the project
+and indicate that the project is aborted[^25]. Deleting and overwriting files is also not an option at this
+stage. This is a design choice and reduces the risk of Researchers downloading incomplete or
+changed data.
+
+### Deleted
+
+A project can only be deleted if it has the status In Progress and it has not been previously released
+to the recipients. The Deleted status is to indicate that something has gone wrong, e.g. project
+information is incorrect.
+When a project is deleted, all data is deleted from the cloud. In addition, all project metadata aside
+from the bucket name is deleted from the database. The name of the bucket is required for invoicing
+purposes and keeping track of that there are no residual files left after deletion. The deletion of a
+project is not reversible - once a project is deleted, it cannot be used for delivery again.
+
+### Available
+
+When a project is released, all roles (aside from Super Admins) have access to the data.
+Researchers can now download the project data. Upload is no longer possible. If additional data
+must be uploaded within the same project, a Unit Admin or Unit Personnel can retract the project,
+thereby setting the status to In Progress again.
+An Available project cannot be deleted. If the project should be closed, a Unit Admin or Unit
+Personnel can do this by archiving the project. To indicate that something has gone wrong and that
+the project life cycle has deviated from the standard, use the abort flag. If nothing has gone wrong
+and the project should simply be closed due to that the data has been downloaded by the users, the
+archivation command can be used as is.
+When a project is released (from In Progress), the countdown of DiA begins. After this number of
+days have passed, the project is automatically expired.
+
+### Expired
+
+The project status Expired can only occur if DiA days pass - a project cannot be expired via the CLI.
+When the status is changed to Expired, the countdown of the DiE number of days starts. In this
+status, all data and metadata is kept in the system, but all uploads, downloads and deletions of data
+by all roles are blocked. After DiE days, the project status is automatically changed to Archived. This
+is the end of a project life cycle and all files are deleted. Metadata, however, is kept.
+In the case that the data recipients do not download the data in time, they can contact one of the Unit
+Admins or Unit Personnel connected to the unit responsible for the project. These roles then have
+the necessary permissions to release the project data again. Once released again, this starts the
+countdown of DiA again.
+The renewal of data access can be performed a maximum of two times. Once a project expires, it
+can be released again, then expire again and then released one last time. After this, the project
+expires, and can then only be archived.
+
+### Archived
+
+An archived project marks the end of a project life cycle. This is not reversible. All data is deleted
+and all metadata is kept.
+
+### Aborted
+
+An archived project can also be marked as aborted. This indicates that data has been (either
+intentionally or accidentally) shared with the recipients and that something has gone wrong during
+the data delivery. It can also indicate that the Unit Admin or Unit Personnel wishes for the project
+metadata to be deleted, which it is not if the project is simply archived.
+
+## C Keys
+
+Information on the user- and project keys are coming.
+
+## D Flowcharts
+
+This section shows flowcharts describing the DDS. The flowcharts are of varying sizes and some
+can therefore be difficult to follow and read. To see these in a better resolution, please
+[see here](https://drive.google.com/file/d/1ophR0vtGByHxPG90mzjAPXgMTCjVcN_Z/view?usp=sharing).
+Note that after clicking the link, you need to click the “open with”, “diagrams.net” and
+then authorize with google to access it. The flowcharts are located in the tab “Flowcharts”, found at
+the bottom of the board. These will most likely be available in a different format later, e.g. not
+requiring access to the diagrams.net page.
+
+\blandscape
+
+### Authentication
+
+
+\elandscape
+
+\blandscape
+
+### Setup of 2FA via Authentication App
+
+
+\elandscape
+
+\blandscape
+
+### Inviting users
+
+
+\elandscape
+
+\blandscape
+
+### Inviting users to Project
+
+
+\elandscape
+
+\blandscape
+
+### Creating a Project
+
+
+\elandscape
+
+\blandscape
+
+### Uploading data
+
+
+\elandscape
+
+\blandscape
+
+### Downloading data
+
+
+\elandscape
+
+[comment]: The footnotes text is stored here. LaTeX figures out how to put them where they're referenced.
+
+[^1]:
+
+[^2]:
+ For more information on Safespring and their storage service, visit
+
+
+[^3]:
+ The storage time is described below: [Automatic expiry of data access and archiving of project](#automatic-expiry-of-data-access-and-archiving-of-project),
+ and in the Appendix - [Project Statuses](#b-project-statuses).
+
+[^4]: The cluster nodes can only be accessed by admins.
+
+[^5]:
+ An invite is valid for 7 days. At this time, there is no clean up functionality to remove unanswered invites,
+ however the invitee can follow the expired invite link to deactivate it completely or the inviter can remove the
+ invite. Both of these alternatives can then be followed by a new invite to the same user if needed.
+
+[^6]:
+
+[^7]:
+
+[^8]:
+ If a custom destination is chosen during authentication you also must specify the path to the token in all
+ following commands.
+
+[^9]:
+ We’re looking into the possibility of authenticating during an ongoing delivery to reduce the risk of a failed
+ delivery due to an expired token. This is not implemented at this time. Therefore, if the delivery is not finished
+ before the token expires, the ongoing upload or download will fail.
+
+[^10]:
+
+[^11]:
+ Unit Admins/Personnel cannot be invited to specific projects - all accounts with these roles have access to
+ all projects within the specific unit.
+
+[^12]:
+
+[^13]:
+ “Data access loss” refers to the occasion that no user connected to a specific unit has access to download
+ and decrypt data within a specific project. Super Admins can not help with this since they do not have access
+ to any of the data.
+
+[^14]:
+
+[^15]:
+
+[^16]:
+
+[^17]: The cleanup actions will be executed once a week, manually by one of the sysadmins.
+
+[^18]:
+ Compression algorithm: ZStandard.
+ See [this link](https://github.com/ScilifelabDataCentre/dds_web/blob/dev/doc/architecture/decisions/0006-use-zstandard-as-the-compression-algorithm.md)
+ for more information on why this was chosen as the compression algorithm.
+
+[^19]: Collection of the Safespring keys is automatically handled by the CLI.
+
+[^20]:
+
+[^21]:
+ When releasing a project, the default DiA is the value set during unit creation in the DDS. It is possible,
+ however, to specify a custom DiA for a specific project by using the deadline option in the release command.
+
+[^22]:
+ This was originally to avoid any overwriting of existing files, however it has been brought to our attention
+ that the ability to specify the same staging directory for several downloads would be a welcome feature. There
+ is a plan to look into this.
+
+[^23]: The project is set to non-active.
+
+[^24]:
+ Note that it may be a good idea to use the –no-mail flag in this case to avoid Researchers receiving email
+ notifications regarding available data.
+
+[^25]:
diff --git a/doc/troubleshooting.md b/doc/troubleshooting.md
new file mode 100644
index 000000000..ca0176a22
--- /dev/null
+++ b/doc/troubleshooting.md
@@ -0,0 +1,492 @@
+---
+title: "Data Delivery System"
+subtitle: "Troubleshooting"
+toc: true
+geometry: "left=1cm, right=1cm, top=2cm, bottom=2cm"
+fontsize: 12pt
+colorlinks: true
+header-includes:
+ # Next 2 lines set the font to Lato
+ - \usepackage{lato}
+ - \renewcommand{\familydefault}{\sfdefault}
+ # Next 3 lines set the header to include the date
+ - \usepackage{fancyhdr}
+ - \pagestyle{fancy}
+ - \rhead{30-Jul-2024}
+---
+
+# Users: Unit Admin/Personnel/Researchers
+
+## A. Invitation link invalid
+
+Did you receive the invitation email more than 7 days ago?
+
+- Yes: An invitation is only valid for 7 days. Contact the SciLifeLab unit that sent the original invitation and ask them to invite you again.
+- No: Something else has gone wrong. Contact the SciLifeLab unit that sent the original invitation and ask them to:
+ 1. Remove the invitation with dds user delete --is-invite
+ 2. Invite you again
+ 3. Inform the Data Centre of the issue.
+
+## B. Too many authentication requests in one hour
+
+If this message is displayed in the web interface or CLI, the user has attempted to log in or logged in and out more
+than 10 times within an hour. Please wait an hour and try again.
+
+## C. Web: Invalid username or password
+
+### 1. Are you sure you are using the correct username?
+
+- Yes: Move on to step 2
+- No: It’s currently not possible to reset/change a password. There are three options depending on your DDS account role:
+ - Unit Admin or Unit Personnel:
+ - Ask a colleague to list the users within your unit (dds user ls) and check the table for the row with your name.
+ - Verify that the username matches the one you are using.
+ - Researcher: Contact the SciLifeLab unit that invited you to the DDS.
+ - Question for the Unit Admins / Personnel: Is the account connected to a project within the DDS?
+ - Yes: A Unit Admin or Unit Personnel role can list the researchers in a specific project: `dds ls --project --users`. They can then check the table for the user and find the correct username.
+ - No: The Unit Admin or Personnel should contact .
+
+### 2. Are you sure you are using the correct password?
+
+- Yes: Notify the Data Centre, .
+- No: Try requesting a password reset in the web interface.
+
+- IMPORTANT: If you have access to any projects before the password reset, notify an Unit Admin or Unit Personnel
+ immediately that they need to run the following command:
+ `dds project access fix`
+
+The user running this command needs to have access to the projects in question. This can be seen in the “Access” column
+when a user does dds ls to list the projects.
+
+## D. CLI: Failed to authenticate user: Missing or incorrect credentials
+
+Follow the same steps as in [C](#c.-web-invalid-username-or-password) above.
+
+## E. Not receiving emails
+
+KI seems to have a spam filter which makes receiving emails very slow in some cases. The email should show up at some
+point, but it may take some time.
+
+If you have an email that you are certain does not go through the KI spam filter:
+
+- Verify that you are checking the correct email address
+- Check the junk folder
+
+If you cannot find the email, contact the Data Centre.
+
+## F. CLI Documentation not accessible
+
+You can download the CLI documentation as a PDF.
+Click [here](https://github.com/ScilifelabDataCentre/dds_cli/releases/latest/download/dds_cli_user_manual.pdf). The
+download should start automatically. Notify the Data Centre that the documentation page is down so that we can look
+into it.
+
+## G. Long error message (traceback)
+
+If you get a long error message after running the CLI:
+
+- Unit Personnel / Admins: Report this to [delivery@scilifelab.se](mailto:delivery@scilifelab.se).
+- Researchers: Report this to the SciLifeLab unit delivering the data.
+
+Include the error message in the email. If the error is related to the DDS, an understandable message should always be
+displayed and it’s therefore important that we get information so that we can change it.
+
+## H. Token expires during delivery
+
+Unfortunately we cannot do anything about this at this time. Remember to authenticate yourself before every large
+delivery. Re-authenticate yourself and then upload or download again. We will look into this issue if and when it arises.
+
+## I. Windows OS: Warning message "Storing the login information locally - please ensure no one else can access the file at …"
+
+Since the token file permissions check does not work on Windows OS, we are warning the users to be extra cautious in
+keeping this file secure. The dds client should be fully functional despite this warning.
+
+## J. Permission denied (authentication token) when using the DDS on a func account or similar
+
+The default setting is that the authenticated token generated by `dds auth login` has permission 600. This means read
+and write permissions for the specific unix user that is running the command.
+
+Running the dds auth login command with the option `--allow-group` will allow users from the same unix group to use the
+resulting token file. However, keep the following in mind:
+
+- The permissions of tokens cannot be changed after the tokens are established. If you began an authenticated session
+ without the use of the `--allow-group` option, but want to use it in a new session, use `dds auth logout` to end the
+ current session. Then use the `--allow-group` option and start a new session. This also applies to the reverse.
+- Not recommended for users with role Researcher, use with care.
+
+## K. QuotaExceeded
+
+The error could look something like this:
+
+```
+raise error_class(parsed_response, operation_name)
+
+ botocore.exceptions.ClientError: An error occurred (QuotaExceeded) when calling the UploadPart operation: Unknown
+
+```
+
+This indicates that the cloud storage location has a specific limit for your unit and that it has been exceeded.
+Contact the Data Centre, inform them of the error (remember to include the full error
+message) and tell them the size of the data you are attempting to upload.
+
+## L. Unrecoverable key error
+
+The most likely reason for a key related error is that you do not have access to the project you are trying to access.
+If you get an error message containing something similar to “Unrecoverable key error”, please go through the following
+steps:
+
+### 1. Run `dds ls`
+
+- Find the row of the project you were trying to access or use
+- Check the “Access” column for that row - does it contain a red cross or a green check mark?
+ - Green check: You have access to the project. This is not the issue.
+ - Does the current project status allow for the action you are trying to perform? Please look at the technical
+ overview and documentation. These are linked at the start of this document.
+ - Yes: Go to step 2.
+ - No: This is most likely the issue. Make sure the project status allows for the current action.
+ Please report the error message, what you were trying to do, the exact command, what steps you have performed
+ and any other information you may have, and we will look into this.
+ - Red cross: You do not have access to the project. Important note: If no one else has access to the project,
+ please note that the Data Centre cannot help you. As Super Admins in the DDS, we do not have access to any of your
+ data and we cannot restore your access. Please read the technical overview, the link is displayed at
+ delivery.scilifelab.se and also when running the CLI commands.
+ - Researcher: You need to contact all SciLifeLab units responsible for the projects you are involved in in the DDS.
+ They need to run `dds project access fix` for you. Try again after that is done.
+ - Unit Admin / Personnel: You need to contact a colleague with access to the projects you are trying to use.
+ They need to run `dds project access fix` for you. Try again after that is done.
+
+### 2. If you have checked the project status, the project access, the access is fixed, or you have not been able to identify the issue, please contact the Data Centre and include the full error message, the exact command you are running.
+
+## M. ERROR: Internal Server Error
+
+Contact [delivery@scilifelab.se](mailto:delivery@scilifelab.se) and provide the information listed [here](https://delivery.scilifelab.se/trouble).
+
+## N. TooManyBuckets
+
+Safespring has a maximum number of buckets per unit, however this number is set quite high and therefore this should
+not happen. If it does happen, contact the Data Centre and we will fix this. You can also check the active projects
+and see if there is one that is an active project that can be archived, for example. Be careful with this and
+do not delete/archive (including abort) projects which you are not 100% sure that they are ready to be deleted/archived.
+This is not reversible.
+
+## O. Errors occurred during upload
+
+When running `dds data put`, if there are issues, the following is a possible output:
+
+```
+Errors occurred during upload.
+
+If you wish to retry the upload, re-run the `dds data put` command again, specifying the same options as you did now. To also overwrite the files that were uploaded, also add the `--overwrite` flag at the end of the command.
+
+See /code/DataDelivery_2022-06-30_14-17-18/logs/dds_failed_delivery.json for more information.
+```
+
+Inform the Data Centre of the issue.
+
+## P. Not possible to change project status (Deadlock)
+
+A while back, someone commented that they were having issues with changing the status of a project. It turned out that
+the status had actually been changed, so before contacting the Data Centre, check the project status and see if it has
+changed despite the error.
+
+## Q. Lost access to 2FA Authenticator App
+
+If you have configured the Two Factor Authentication of your DDS account to use an authenticator app, but you have
+lost access to it (e.g. due to a broken or lost device), the Data Centre can deactivate the configuration in order to
+let you authenticate using the default email option. After this you can choose to activate the authenticator app method
+again.
+
+- Researcher role: Please contact a SciLifeLab unit responsible for one of your ongoing delivery projects and tell them
+ you need a reset of the 2FA method.
+- Unit Admin / Personnel: Please contact Data Centre and ask for a reset of the 2FA method. Inform them of the username
+ and email address of the affected account.
+
+## R. Windows: DDS freezes after message
+
+The output from the DDS could look something like this:
+
+```
+PS Z:\> dds auth login
+dds : \ufe35
+At line:1 char:1
++ dds auth login
++ ~~~~~~~~~~~~~~
++ CategoryInfo : NotSpecified: ( \ufe35 :String) [], RemoteException
++ FullyQualifiedErrorId : NativeCommandError
+\ufe35 ( ) \ufe35
+( ) ) ( ( ) SciLifeLab Data Delivery System
+\ufe36 ( ) ) ( https://delivery.scilifelab.se/
+\ufe36 ( ) Version 2.1.1
+\ufe36
+
+INFO Attempting to create the session token
+```
+
+Unfortunately this seems to have something to do with the Windows command prompt and the encoding (or logging), and not
+the DDS. We have attempted to reproduce the issue in both the Command Prompt and Powershell without success. To attempt
+to solve this, look at the following steps:
+
+### How did you install the CLI?
+
+- From PyPi, with the instructions found [here](https://scilifelabdatacentre.github.io/dds_cli/installation/#install-from-pypi).
+ - Please download the executable, following the instructions, and try running the same command again (but with the executable).
+- I downloaded the executable as instructed [here](https://scilifelabdatacentre.github.io/dds_cli/installation/#install-via-the-executable).
+ - Inform the Data Centre.
+
+Depending on your level of experience with Stack Overflow and the command line, [this comment](https://stackoverflow.com/a/20950421)
+could potentially help you to solve the issue.
+
+## S. The download is interrupted
+
+### 1. Are you attempting to download the data within Sweden?
+
+- Yes, I’m in Sweden: Move on to 2.
+- No, I’m not in Sweden: The DDS uploads data to a cloud service located within the Swedish borders. Downloading data
+ overseas is most likely slower, however it should work. Verify that your internet connection is stable. Note that
+ downloading data via a hotel WiFi connection a) is not recommended, and b) will most likely not be permitted due to hotel
+ WiFi limits.
+
+### 2. Have you checked that you have enough storage space available in the location you’re attempting to download the data?
+
+- Yes, I have enough space: Move on to 3.
+- No: Please make sure you have enough storage space available.
+
+### 3. Contact the SciLifeLab unit delivering your data.
+
+## T. “No data specified”
+
+This message is displayed when you have not specified what data to download. In order to download data from your project,
+you need to use one of the following options:
+
+`--source`/`–s`: Downloads a specific file or directory. You can use this option multiple times to specify separate
+files or directories.
+
+`--get-all`/`–a`: Downloads the entire content of the project.
+
+# Data Centre
+
+## A. Invitation link invalid
+
+Ask them to follow the instructions in the Invitation link invalid section in the User part of this document.
+Unit Admins/Personnel can also run the following commands:
+
+- Delete the invite if it exists: `dds user delete --is-invite`. If they are experiencing issues with this, we as
+ Super Admins can also do this.
+- Invite them again. Super Admins should not be doing this.
+
+## B. Too many authentication requests in one hour
+
+If this message is displayed in the web interface or CLI, the user has attempted to log in or logged in and out more
+than 10 times within an hour. Ask the user to wait for an hour and try again later.
+
+## C. Web: Invalid username or password
+
+If a user cannot log in to the web interface, ask them to follow the steps in [C above](#c.-web-invalid-username-or-password).
+If they have already done this, check that they have provided all the information listed [here](https://delivery.scilifelab.se/trouble).
+
+A possibility is that the web is not letting them in because they have attempted to log in more than 10 times in an hour.
+If that’s the case, tell them to try again later. If it’s not the case, there’s not much you can do. Offer to delete their
+account and invite them again. It’s better if a Unit Admin or Personnel handles the new invite, but if you do, remember
+that you cannot invite Researchers to specific projects, only to the DDS in general and that you need to specify the unit
+public ID if you want to invite a Unit Personnel or Admin. However, if there are active projects in the unit, they will
+still not have access, so they still need to ask someone within their unit with access to the projects, for renewal of
+the project access.
+
+## D. CLI: Failed to authenticate user: Missing or incorrect credentials
+
+Follow the same steps as in [the above C](#c.-web-invalid-username-or-password-1) above.
+
+## E. Not receiving emails
+
+Check the information in the User section earlier in the document regarding this specific topic. If they contact us,
+the email is correct and it’s not an email address connected to the KI spam filter, create a card in the Jira board.
+
+## F. CLI Documentation not accessible
+
+1. Notify the users that they can download the CLI documentation as a PDF from [here](https://github.com/ScilifelabDataCentre/dds_cli/releases/latest/download/dds_cli_user_manual.pdf)
+2. Investigate the issue, including whether it could be infrastructure-related.
+
+## G. Long error message (traceback)
+
+1. Make sure they have upgraded the CLI to the latest version. If they have, or the upgrade does not help, move on to 2.
+2. Make sure they have provided the information listed [here](https://delivery.scilifelab.se/trouble).
+3. Look through the traceback. There could be some uncaught error or similar in the CLI, most often this is not caused
+ by the backend, and most likely this is not something that can wait. If that’s the case, create a card in Jira.
+
+## H. Token expires during delivery
+
+We cannot do anything about this. It’s on the todo list. No need to add a card about this, but add a comment to the
+existing ticket (DDS-1173).
+
+## I. Windows OS: Warning message "Storing the login information locally - please ensure no one else can access the file at …"
+
+Since the token file permissions check does not work on Windows OS, we are warning the users to be extra cautious in
+keeping this file secure. The dds client should be fully functional despite this warning.
+
+## J. Permission denied (authentication token) when using the DDS on a func account or similar
+
+### Basic information
+
+- The default setting is that the authenticated token generated by `dds auth login` has permission 600. This means read
+ and write permissions for the specific unix user that is running the command.
+- Running the `dds auth login` command with the option `--allow-group` will allow users from the same unix group to use
+ the resulting token file.
+- The permissions of tokens cannot be changed after the tokens are established. If you began an authenticated session
+ without the use of the `--allow-group option`, but want to use it in a new session, use `dds auth logout` to end the
+ current session. Then use the `--allow-group option` and start a new session. This also applies to the reverse.
+- Not recommended for users with role Researcher, use with care.
+- As long as the `dds auth login` command works without the additional permissions option, that is what the users will
+- have to work with for now.
+
+### Background / additional information:
+
+Information from NGI and explanation to our (possibly temporary) solution: On Miarka, the deliveries are done through a
+web service which is run as a special user (func account) and the access for that user is heavily restricted. Usually
+the bioinformaticians that perform the data deliveries do not have access to that account and instead the delivery is
+done through API calls to the service. When starting the integration testing, their idea for the delivery with the DDS
+was that a user would generate an authentication token with their account on Miarka, place it in a folder that is
+accessible to both them and the func account, and then tell the DDS, run by the func account, to use that authentication
+token.
+
+When performing the authentication, the DDS CLI originally collected the token from the API and saved that in a file
+with permission 600: `-rw-------` (only read and write for that specific user). Because of the issue mentioned above,
+there was a suggestion to change this to allow permission 660: `-rw-rw----` (read and write for specific user and group).
+In the end, we have now added the option `--allow-group` as a flag to the `dds auth login` command so that they can
+(should be able to but as of writing this it has not yet been tested) allow the func account to use the authenticated
+token for deliveries with the DDS. However, the permissions are now set to 640: `-rw-r-----` (read and write permissions
+for the user that is using the command and only read permission for a group). This is a conscious choice since the func
+account should only need to read the token file and should not be able to edit it, etc.
+
+We have discussed the potential problems with this and we will look into it again when we can. For the time being we
+needed the DDS to be usable and therefore this is implemented. We will look into this again at some point and it is
+possible that this will change in some way, we do not yet know if and in that case how.
+
+## K. QuotaExceeded
+
+The error could end with something like this:
+
+```
+raise error_class(parsed_response, operation_name)
+
+ botocore.exceptions.ClientError: An error occurred (QuotaExceeded) when calling the UploadPart operation: Unknown
+```
+
+Hopefully we should be catching these and displaying something more user friendly, but if the error is not caught,
+then do the following:
+
+- Check with the user (if they have not already included the information in their email) which unit they have an account
+ within and the size of the data that they are attempting to upload.
+- Contact Safespring (will post the email to them and other important information in the dds-development channel on
+ slack) and ask them to increase the quota for the specific project. It should currently be set to 100TB for all NGI units.
+ It doesn’t matter which value you choose when you ask Safespring to increase it, it’s just there as a precaution so that
+ data is not uploaded without someone’s knowledge. This is not something that we can affect, it’s on Safespring's side.
+- When the quota has been changed, inform the users that they can try the upload again.
+
+## L. Unrecoverable key error
+
+The most likely reason for a key related error is that the user does not have access to the project they are trying to
+access, and that there is an uncaught error somewhere. Add it as a card to the Jira board.
+
+1. Ask the user to perform the steps in the corresponding user section. If they have done this, check if the user is
+ active or not first: `dds user ls`. This should display all users and the right-most column (“Active”) shows if they
+ are active (green check mark) or not (red cross). If they are inactive, then that’s the issue. There shouldn’t be a
+ possibility of this causing this type of error, report it as a bug in the Jira board. Inform the user that someone in
+ their unit should activate them.
+2. Ask them to run the commands as specified in the Unrecoverable key error section earlier in the document and for them
+ to send you the output of those commands. Does the current project status allow for the command they are trying to run?
+ Does it say that they have access to the project?
+3. Ask them to create a new project and to do the same command with that one. If it works, the issue is with that specific
+ project and this should be reported as a bug in the Jira board.
+
+If, when running the `dds project access fix` commands, the users get error messages, it’s most likely that the user
+does not have access to the projects. If no one else has access to the project either, we cannot help. This information
+is clearly explained in the technical overview: As Super Admins in the DDS, we do not have access to any of the data and
+we cannot restore any access. They will need to create a completely new project and possibly perform the delivery a
+second time.
+
+## M. ERROR: Internal Server Error
+
+The most likely reason for an Internal Server Error is an uncaught exception in the backend, often something database
+related. If someone contacts you regarding an Internal Server Error, begin with making sure they have provided the
+information mentioned [here](https://delivery.scilifelab.se/trouble). When they have done this,
+look through the traceback.
+
+If you cannot find the issue in the traceback, get the logs from the backend. The log should not be posted in Github
+just in case of the logs showing anything that shouldn’t be public information (this shouldn’t happen but it’s a precaution).
+
+## N. TooManyBuckets
+
+Safespring has a maximum number of buckets per unit, however this number is set quite high and therefore this should
+not happen. If it does happen, it should only be possible when a user is creating a project. It’s also possible that
+this is the cause of a failed project creation, so if a project cannot be created and there is a questionable error
+message displayed, check the logs and see if the bucket could be created. To increase the number
+of allowed buckets, contact Safespring and ask them to increase the number of allowed buckets in the affected Safespring
+project, listed and pinned in the dds-development channel on Slack. After this, the users can attempt creating a project
+again.
+
+## O. Errors occurred during upload
+
+The message from the CLI could be something like this:
+
+```
+Errors occurred during upload.
+If you wish to retry the upload, re-run the `dds data put` command again, specifying the same options as you did now. To also overwrite the files that were uploaded, also add the `--overwrite` flag at the end of the command.
+
+
+See /code/DataDelivery_2022-06-30_14-17-18/logs/dds_failed_delivery.json for more information.
+```
+
+Note: Normally we would need to do the following, but as is noted in the section in the User part above, they can simply
+perform the upload again at this time. This has also not happened yet apart from during the testing phase, but there’s
+a first time for everything.
+
+If a user contacts us with an upload issue that has generated the aforementioned file, do the following:
+
+1. Ask them to send the exact CLI command that they ran
+2. Get the JSON file.
+3. Download that file and run the following command in the cluster, in the DDS backend:
+ `flask update-uploaded-file` while specifying the project that the user has used in the command emailed to us, and the
+ path to the file as the options. This command goes through the file and checks if the file has been uploaded but has
+ not been added to the database. If so, the backend adds the file information to the database which should mean that
+ the project contents should now be accessible for download.
+
+Make sure to do this as soon as possible since it’s possible that the upload has in fact been successful but the
+information has not been added to the database. This means that the data is located in the cloud and will take up space
+there, but you cannot access it since it is not recorded in the database.
+
+## P. Not possible to change project status (Deadlock)
+
+A while back, someone commented that they were having issues with changing the status of a project. It turned out that
+the status had actually been changed, but that there was a deadlock happening somewhere in the backend. Ask the user
+to check the project status if they haven’t already, it may be the status they were trying to change to. Otherwise, at
+this point the only thing we can do is ask them to wait for a bit and then try again, and tell them that we know this
+may happen and that we are looking into it.
+
+## Q. Lost access to 2FA Authenticator App
+
+If you have configured the Two Factor Authentication of your DDS account to use an authenticator app, but you have lost
+access to it (e.g. due to a broken or lost device), another Super Admin can deactivate the configuration in order to
+let you authenticate using the default email option. After this you can choose to activate the authenticator app method
+again. Super Admins can also do this for other account roles.
+
+Run the following command:
+
+`dds auth twofactor deactivate`
+
+## R. Windows: DDS freezes after message
+
+Ask them to follow the corresponding User section of this document. If they have done that already and it does not help,
+there’s nothing we can do. There’s a link in the section to a command they could run, but since we haven’t been able to
+reproduce it, we cannot guarantee that it would work. Usually it works if they use the executable instead of the CLI
+installed via PyPi.
+
+## S. The download is interrupted
+
+All cases reported so far have not been connected to the DDS specifically. It’s most likely due to a network issue or
+lack of storage space as described in the corresponding section in the User section. Make sure that they provide the
+information specified on the and create an issue in Jira. It will have to wait unfortunately.
+
+## T. “No data specified”
+
+The user has not specified what they wish to download. Make sure that the user has specified either `--source`/`–s`
+or `--get-all`/`–a` as described in the user section.
diff --git a/requirements.txt b/requirements.txt
index 9ed7512e8..b31942161 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,7 +8,7 @@ blinker==1.4
boto3==1.20.47
botocore==1.23.47
cachetools==5.2.0
-certifi==2023.07.22
+certifi==2024.7.4
cffi==1.15.0
charset-normalizer==2.0.11
click==8.0.3
@@ -53,7 +53,7 @@ pytz==2021.3
pytz-deprecation-shim==0.1.0.post0
qrcode==7.3.1
redis==4.5.5
-requests==2.31.0
+requests==2.32.0
requests_cache==0.9.4
rich==12.5.1
s3transfer==0.5.1
@@ -66,5 +66,5 @@ tzlocal==4.1
urllib3==1.26.18
visitor==0.1.3
Werkzeug==3.0.3
-wrapt==1.13.3
+wrapt==1.14.0
WTForms==3.0.1
\ No newline at end of file
diff --git a/tests/requirements-test.txt b/tests/requirements-test.txt
index 4d668030c..b73ddaf5b 100644
--- a/tests/requirements-test.txt
+++ b/tests/requirements-test.txt
@@ -4,6 +4,6 @@ pytest-cov==3.0.0
requests-mock==1.9.3
sqlalchemy-utils==0.38.2
pytest-mock==3.7.0
-pyfakefs==4.5.5
+pyfakefs==5.3.0
requests_cache==0.9.4
testfixtures==7.0.4
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 017a6dfde..cfb4de2b0 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -36,7 +36,8 @@
monthly_usage,
collect_stats,
lost_files_s3_db,
- update_unit,
+ update_unit_sto4,
+ update_unit_quota,
send_usage,
)
from dds_web.database import models
@@ -276,10 +277,10 @@ def test_create_new_unit_success(client, runner, capfd: LogCaptureFixture) -> No
assert new_unit.warning_level
-# update_unit
+# update_unit_sto4
-def test_update_unit_no_such_unit(client, runner, capfd: LogCaptureFixture) -> None:
+def test_update_unit_sto4_no_such_unit(client, runner, capfd: LogCaptureFixture) -> None:
"""Try to update a non existent unit -> Error."""
# Create command options
command_options: typing.List = [
@@ -296,7 +297,7 @@ def test_update_unit_no_such_unit(client, runner, capfd: LogCaptureFixture) -> N
]
# Run command
- result: click.testing.Result = runner.invoke(update_unit, command_options)
+ result: click.testing.Result = runner.invoke(update_unit_sto4, command_options)
assert result.exit_code == 0
assert not result.output
@@ -349,7 +350,7 @@ def test_update_unit_sto4_start_time_exists_mock_prompt_False(
# Run command
# Mock rich prompt - False
with patch.object(rich.prompt.Confirm, "ask", return_value=False) as mock_ask:
- result: click.testing.Result = runner.invoke(update_unit, command_options)
+ result: click.testing.Result = runner.invoke(update_unit_sto4, command_options)
assert result.exit_code == 0
assert not result.output
mock_ask.assert_called_once
@@ -414,7 +415,7 @@ def test_update_unit_sto4_start_time_exists_mock_prompt_True(
# Run command
# Mock rich prompt - True
with patch.object(rich.prompt.Confirm, "ask", return_value=True) as mock_ask:
- result: click.testing.Result = runner.invoke(update_unit, command_options)
+ result: click.testing.Result = runner.invoke(update_unit_sto4, command_options)
assert result.exit_code == 0
assert not result.output
mock_ask.assert_called_once
@@ -443,6 +444,109 @@ def test_update_unit_sto4_start_time_exists_mock_prompt_True(
]
+# update_unit_quota
+
+
+def test_update_unit_quota_no_such_unit(client, runner, capfd: LogCaptureFixture) -> None:
+ """Try to update a non existent unit -> Error."""
+ # Create command options
+ command_options: typing.List = [
+ "--unit-id",
+ "unitdoesntexist",
+ "--quota",
+ 2, # 2 GB,
+ ]
+
+ # Run command
+ result: click.testing.Result = runner.invoke(update_unit_quota, command_options)
+ assert result.exit_code == 1
+ assert not result.output
+
+ # Get logging
+ _, err = capfd.readouterr()
+
+ # Verify message
+ assert f"There is no unit with the public ID '{command_options[1]}'." in err
+
+
+def test_update_unit_quota_confirm_prompt_False(client, runner, capfd: LogCaptureFixture) -> None:
+ """Unit quota should not be changed when answer to prompt is False."""
+ # Get existing unit
+ unit: models.Unit = models.Unit.query.first()
+ unit_id: str = unit.public_id
+
+ # save original quota
+ quota_original = unit.quota
+
+ # Create command options
+ command_options: typing.List = [
+ "--unit-id",
+ unit_id,
+ "--quota",
+ 2, # 2 GB,
+ ]
+
+ # Run command
+ # Mock rich prompt - False
+ with patch.object(rich.prompt.Confirm, "ask", return_value=False) as mock_ask:
+ result: click.testing.Result = runner.invoke(update_unit_quota, command_options)
+ assert result.exit_code == 0
+ assert not result.output
+ mock_ask.assert_called_once
+
+ # Get logging
+ _, err = capfd.readouterr()
+
+ # Verify logging
+ assert f"Cancelling quota update for unit '{unit_id}'." in err
+ assert f"Unit '{unit_id}' updated successfully" not in err
+
+ # Verify no change in unit
+ unit: models.Unit = models.Unit.query.filter_by(public_id=unit_id).first()
+ assert unit
+ assert unit.quota == quota_original
+
+
+def test_update_unit_quota_confirm_prompt_true(client, runner, capfd: LogCaptureFixture) -> None:
+ """Unit quota successfully updated when answer to the prompt is True."""
+
+ # Get existing unit
+ unit: models.Unit = models.Unit.query.first()
+ unit_id: str = unit.public_id
+
+ # save original quota
+ quota_original = unit.quota
+
+ # Create command options
+ command_options: typing.List = [
+ "--unit-id",
+ unit_id,
+ "--quota",
+ 2, # 2 GB,
+ ]
+
+ # Run command
+ # Mock rich prompt - True
+ with patch.object(rich.prompt.Confirm, "ask", return_value=True) as mock_ask:
+ result: click.testing.Result = runner.invoke(update_unit_quota, command_options)
+ assert result.exit_code == 0
+ assert not result.output
+ mock_ask.assert_called_once
+
+ # Get logging
+ _, err = capfd.readouterr()
+
+ # Verify logging
+ assert f"Cancelling quota update for unit '{unit_id}'." not in err
+ assert f"Unit '{unit_id}' updated successfully" in err
+
+ # Verify change in unit
+ unit: models.Unit = models.Unit.query.filter_by(public_id=unit_id).first()
+ assert unit
+ assert unit.quota != quota_original
+ assert unit.quota == command_options[3] * 1000**3 # GB to bytes
+
+
# update_uploaded_file_with_log
diff --git a/tests/test_files_list.py b/tests/test_files_list.py
index d294093e5..3eae0d213 100644
--- a/tests/test_files_list.py
+++ b/tests/test_files_list.py
@@ -180,6 +180,38 @@ def test_list_files_auth(client):
)
+@pytest.mark.parametrize("root_path", ["/", "./", "."])
+def test_list_files_root_path(client, root_path):
+ """Test listing files when different representations of the root path are requested."""
+ # Common setup: Make project available
+ response = client.post(
+ tests.DDSEndpoint.PROJECT_STATUS,
+ headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client),
+ query_string={"project": "public_project_id"},
+ json={"new_status": "Available"},
+ )
+ assert response.status_code == http.HTTPStatus.OK
+
+ # Test using parameterized root path
+ response = client.get(
+ tests.DDSEndpoint.LIST_FILES,
+ headers=tests.UserAuth(tests.USER_CREDENTIALS["researchuser"]).token(client),
+ query_string={"project": "public_project_id"},
+ json={"subpath": root_path},
+ )
+
+ expected_files_folders = [
+ {"folder": True, "name": "filename1"},
+ {"folder": True, "name": "filename2"},
+ {"folder": True, "name": "sub"},
+ ]
+
+ assert "files_folders" in response.json
+ assert len(response.json["files_folders"]) == len(expected_files_folders)
+ for entry in response.json["files_folders"]:
+ assert entry in expected_files_folders
+
+
def test_list_files_database_error_items(client):
"""List files endpoint raises DB error in query"""
diff --git a/tests/test_version.py b/tests/test_version.py
index 92c2234cb..ff0e1562e 100644
--- a/tests/test_version.py
+++ b/tests/test_version.py
@@ -2,4 +2,4 @@
def test_version():
- assert version.__version__ == "2.7.1"
+ assert version.__version__ == "2.8.0"
diff --git a/tests/tests_v3/test_files_list.py b/tests/tests_v3/test_files_list.py
index 4ba98a4e9..8ff24798b 100644
--- a/tests/tests_v3/test_files_list.py
+++ b/tests/tests_v3/test_files_list.py
@@ -6,6 +6,9 @@
# Own
import tests.tests_v3 as tests
+#
+import pytest
+
# CONFIG ################################################################################## CONFIG #
proj_data = {"pi": "piName", "title": "Test proj", "description": "A longer project description"}
@@ -196,6 +199,37 @@ def test_list_files_auth(client):
)
+@pytest.mark.parametrize("root_path", ["/", "./", "."])
+def test_list_files_root_path(client, root_path):
+ """Test listing files when different representations of the root path are requested."""
+ # Common setup: Make project available
+ response = client.post(
+ tests.DDSEndpoint.PROJECT_STATUS,
+ headers=tests.UserAuth(tests.USER_CREDENTIALS["unituser"]).token(client),
+ query_string={"project": "public_project_id"},
+ json={"new_status": "Available"},
+ )
+ assert response.status_code == http.HTTPStatus.OK
+
+ # Test using parameterized root path
+ response = client.get(
+ tests.DDSEndpoint.LIST_FILES,
+ headers=tests.UserAuth(tests.USER_CREDENTIALS["researchuser"]).token(client),
+ query_string={"project": "public_project_id", "subpath": root_path},
+ )
+
+ expected_files_folders = [
+ {"folder": True, "name": "filename1"},
+ {"folder": True, "name": "filename2"},
+ {"folder": True, "name": "sub"},
+ ]
+
+ assert "files_folders" in response.json
+ assert len(response.json["files_folders"]) == len(expected_files_folders)
+ for entry in response.json["files_folders"]:
+ assert entry in expected_files_folders
+
+
def test_list_project_with_no_files(client):
"""List project with no files"""
-
+
diff --git a/SPRINTLOG.md b/SPRINTLOG.md
index d0efaacc9..c8c7c7d62 100644
--- a/SPRINTLOG.md
+++ b/SPRINTLOG.md
@@ -231,11 +231,11 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe
# 2023-03-17 - 2023-03-31
-_Nothing merged in CLI during this sprint_
+_Nothing merged during this sprint_
# 2023-03-31 - 2023-04-14
-_Nothing merged in CLI during this sprint_
+_Nothing merged during this sprint_
# 2023-04-14 - 2023-04-28
@@ -404,3 +404,29 @@ _Nothing merged in CLI during this sprint_
- Update pymysql to address cve ([#1534](https://github.com/ScilifelabDataCentre/dds_web/pull/1534))
- Update authlib to address cve ([#1535](https://github.com/ScilifelabDataCentre/dds_web/pull/1535))
- Update node packages to address cve ([#1536](https://github.com/ScilifelabDataCentre/dds_web/pull/1536))
+
+# 2024-07-15 - 2024-07-26
+
+- Move raw Technical Overview doc to repo, add page numbers ([#1539](https://github.com/ScilifelabDataCentre/dds_web/pull/1539))
+- Small updates to Technical Overview contents ([#1540](https://github.com/ScilifelabDataCentre/dds_web/pull/1540))
+- Build Technical Overview PDF in GitHub Actions, rename to include DDS and remove option to view on GitHub ([#1541](https://github.com/ScilifelabDataCentre/dds_web/pull/1541/))
+- Fixed index out of range when listing files from root ([#1543](https://github.com/ScilifelabDataCentre/dds_web/pull/1543/))
+- Update Trivy GitHub Actions ([#1545](https://github.com/ScilifelabDataCentre/dds_web/pull/1545))
+
+# 2024-07-29 - 2024-08-09
+
+- Move raw troubleshooting doc to repo and make small updates ([#1546](https://github.com/ScilifelabDataCentre/dds_web/pull/1546))
+
+# 2024-08-12 - 2024-08-23
+
+_Nothing merged during this sprint_
+
+# 2024-08-26 - 2024-09-06
+
+- Update certifi to remove GLOBALISSUER certicates ([#1549](https://github.com/ScilifelabDataCentre/dds_web/pull/1549))
+- Add CODEOWNERS file in order to define Team Hermes as owners of all files in repository ([#708](https://github.com/ScilifelabDataCentre/dds_web/pull/708))
+
+# 2024-09-09 - 2024-09-20
+
+- Flask command to update unit quotas ([#1551](https://github.com/ScilifelabDataCentre/dds_web/pull/1551))
+- Bump python base image to 3.12 and related libraries in both web and client([#1548](https://github.com/ScilifelabDataCentre/dds_web/pull/1548))
diff --git a/dds_web/__init__.py b/dds_web/__init__.py
index cb63b3faa..5afd64728 100644
--- a/dds_web/__init__.py
+++ b/dds_web/__init__.py
@@ -282,13 +282,15 @@ def load_user(user_id):
send_usage,
collect_stats,
monitor_usage,
- update_unit,
+ update_unit_sto4,
+ update_unit_quota,
)
# Add flask commands - general
app.cli.add_command(fill_db_wrapper)
app.cli.add_command(create_new_unit)
- app.cli.add_command(update_unit)
+ app.cli.add_command(update_unit_sto4)
+ app.cli.add_command(update_unit_quota)
app.cli.add_command(update_uploaded_file_with_log)
app.cli.add_command(lost_files_s3_db)
diff --git a/dds_web/api/files.py b/dds_web/api/files.py
index 32953ed2b..27ec6a6e7 100644
--- a/dds_web/api/files.py
+++ b/dds_web/api/files.py
@@ -291,71 +291,30 @@ class ListFiles(flask_restful.Resource):
def get(self):
"""Get a list of files within the specified folder."""
+ # Verify project ID and access
+ project = project_schemas.ProjectRequiredSchema().load(flask.request.args)
+
+ if auth.current_user().role == "Researcher" and project.current_status == "In Progress":
+ raise AccessDeniedError(
+ message="The project status must be 'Available'. Please wait for unit to release the data."
+ )
+
if "api/v1" in flask.request.path:
# requests comming from api/v1 should be handled as before
- return self.old_get()
+ return self.old_get(project)
elif "api/v3" in flask.request.path:
- # Verify project ID and access
- project = project_schemas.ProjectRequiredSchema().load(flask.request.args)
-
- if auth.current_user().role == "Researcher" and project.current_status == "In Progress":
- raise AccessDeniedError(
- message="The project status must be 'Available'. Please wait for unit to release the data."
- )
# Check if to return file size
show_size = flask.request.args.get("show_size", type=inputs.boolean, default=False)
# Check if to get from root or folder
- subpath = "."
- if flask.request.args.get("subpath"):
- subpath = flask.request.args.get("subpath").rstrip(os.sep)
-
- files_folders = list()
-
- # Check project not empty
- if project.num_files == 0:
- return {"num_items": 0, "message": f"The project {project.public_id} is empty."}
+ subpath = flask.request.args.get("subpath", default=".").rstrip(os.sep)
+ subpath = "." if subpath == "" else subpath
+ return self.get_files_folders(project, subpath, show_size)
- # Get files and folders
- distinct_files, distinct_folders = self.items_in_subpath(
- project=project, folder=subpath
- )
-
- # Collect file and folder info to return to CLI
- if distinct_files:
- for x in distinct_files:
- info = {
- "name": x[0] if subpath == "." else x[0].split(os.sep)[-1],
- "folder": False,
- }
- if show_size:
- info.update({"size": float(x[1])})
- files_folders.append(info)
- if distinct_folders:
- for x in distinct_folders:
- info = {
- "name": x if subpath == "." else x.split(os.sep)[-1],
- "folder": True,
- }
-
- if show_size:
- folder_size = self.get_folder_size(project=project, folder_name=x)
- info.update({"size": float(folder_size)})
- files_folders.append(info)
-
- return {"files_folders": files_folders}
-
- def old_get(self):
+ def old_get(self, project):
"""Implementation of old get method. Should be removed when api/v1 is removed."""
- # Verify project ID and access
- project = project_schemas.ProjectRequiredSchema().load(flask.request.args)
-
- if auth.current_user().role == "Researcher" and project.current_status == "In Progress":
- raise AccessDeniedError(
- message="The project status must be 'Available'. Please wait for unit to release the data."
- )
extra_args = flask.request.get_json(silent=True)
if extra_args is None:
@@ -365,16 +324,17 @@ def old_get(self):
show_size = extra_args.get("show_size")
# Check if to get from root or folder
- subpath = "."
- if extra_args.get("subpath"):
- subpath = extra_args.get("subpath").rstrip(os.sep)
+ subpath = extra_args.get("subpath", ".").rstrip(os.sep)
+ subpath = "." if subpath == "" else subpath
- files_folders = list()
+ return self.get_files_folders(project, subpath, show_size)
- # Check project not empty
+ def get_files_folders(self, project, subpath, show_size):
if project.num_files == 0:
return {"num_items": 0, "message": f"The project {project.public_id} is empty."}
+ files_folders = list()
+
# Get files and folders
distinct_files, distinct_folders = self.items_in_subpath(project=project, folder=subpath)
diff --git a/dds_web/commands.py b/dds_web/commands.py
index eca546554..7cd3d8777 100644
--- a/dds_web/commands.py
+++ b/dds_web/commands.py
@@ -157,15 +157,15 @@ def create_new_unit(
gc.collect()
-@click.command("update-unit")
+@click.command("update-unit-sto4")
@click.option("--unit-id", "-u", type=str, required=True)
@click.option("--sto4-endpoint", "-se", type=str, required=True)
@click.option("--sto4-name", "-sn", type=str, required=True)
@click.option("--sto4-access", "-sa", type=str, required=True)
@click.option("--sto4-secret", "-ss", type=str, required=True)
@flask.cli.with_appcontext
-def update_unit(unit_id, sto4_endpoint, sto4_name, sto4_access, sto4_secret):
- """Update unit info."""
+def update_unit_sto4(unit_id, sto4_endpoint, sto4_name, sto4_access, sto4_secret):
+ """Update unit sto4 storage info."""
# Imports
import rich.prompt
from dds_web import db
@@ -205,6 +205,43 @@ def update_unit(unit_id, sto4_endpoint, sto4_name, sto4_access, sto4_secret):
gc.collect()
+@click.command("update-unit-quota")
+@click.option("--unit-id", "-u", type=str, required=True)
+@click.option("--quota", "-q", type=int, required=True)
+@flask.cli.with_appcontext
+def update_unit_quota(unit_id, quota):
+ """Update unit quota. The input is in GB."""
+ # Imports
+ import rich.prompt
+ from dds_web import db
+ from dds_web.database import models
+
+ # Get unit
+ unit: models.Unit = models.Unit.query.filter_by(public_id=unit_id).one_or_none()
+ if not unit:
+ flask.current_app.logger.error(f"There is no unit with the public ID '{unit_id}'.")
+ sys.exit(1)
+
+ # ask the user for confirmation
+ do_update = rich.prompt.Confirm.ask(
+ f"Current quota for unit '{unit_id}' is {round(unit.quota / 1000 ** 3,2)} GB. \n"
+ f"You are about to update the quota to {quota} GB ({quota * 1000 ** 3} bytes). \n"
+ "Are you sure you want to continue?"
+ )
+ if not do_update:
+ flask.current_app.logger.info(
+ f"Cancelling quota update for unit '{unit_id}'. The quota is still {round(unit.quota / 1000 ** 3,2)} GB. ({unit.quota} bytes.)"
+ )
+ return
+
+ # Set sto4 info
+ quota_bytes = quota * 1000**3
+ unit.quota = quota_bytes
+ db.session.commit()
+
+ flask.current_app.logger.info(f"Unit '{unit_id}' updated successfully")
+
+
@click.command("update-uploaded-file")
@click.option("--project", "-p", type=str, required=True)
@click.option("--path-to-log-file", "-fp", type=str, required=True)
diff --git a/dds_web/templates/technical_overview.html b/dds_web/templates/technical_overview.html
index b9a149ae3..388752e1a 100644
--- a/dds_web/templates/technical_overview.html
+++ b/dds_web/templates/technical_overview.html
@@ -3,8 +3,7 @@
{% block body %}