Merge pull request #1432 from ScilifelabDataCentre/dev

New release: Stats
ScilifelabDataCentre · Jun 7, 2023 · 6a5a345 · 6a5a345
2 parents 3c8a734 + af62fbc
commit 6a5a345
Show file tree

Hide file tree

Showing 17 changed files with 416 additions and 23 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -1,6 +1,16 @@
 Changelog
 ==========
 
+.. _2.3.0: 
+
+2.3.0 - 2023-06-07
+~~~~~~~~~~~~~~~~~~~
+
+- Changed the reporting command (cronjob) and added statistics calculations: 
+    - Number of users in total and in different roles
+    - Number of projects: Total, active and non-active
+    - Amount of data (in TBs) currently stored and uploaded since start
+
 .. _2.2.62:
 
 2.2.62 - 2023-03-20

diff --git a/SPRINTLOG.md b/SPRINTLOG.md
@@ -241,3 +241,17 @@ _Nothing merged in CLI during this sprint_
 
 - Documentation: Minor update of Technical Overview ((#1411)[https://github.com/ScilifelabDataCentre/dds_web/pull/1411])
 - Documentation: Account roles and their permissions ((#1412)[https://github.com/ScilifelabDataCentre/dds_web/pull/1412])
+
+# 2023-05-26 - 2023-06-09
+
+- Command:
+  - Save number of Unit Personnel instead of total number of unit users ([#1417](https://github.com/ScilifelabDataCentre/dds_web/pull/1417))
+  - Save total number of projects ([#1418](https://github.com/ScilifelabDataCentre/dds_web/pull/1418))
+  - Save number of Unit Admins ([#1419](https://github.com/ScilifelabDataCentre/dds_web/pull/1419))
+  - Save number of active projects ([#1423](https://github.com/ScilifelabDataCentre/dds_web/pull/1423))
+  - Change `researchuser_count` column name to `researcher_count` in Reporting table ([#1420](https://github.com/ScilifelabDataCentre/dds_web/pull/1420))
+  - Save number of inactive projects ([#1426](https://github.com/ScilifelabDataCentre/dds_web/pull/1426))
+  - Save number of unique Project Owners ([#1421](https://github.com/ScilifelabDataCentre/dds_web/pull/1421))
+  - Save amount of TB's currently stored in system ([#1424](https://github.com/ScilifelabDataCentre/dds_web/pull/1424))
+  - Save amount of TB's uploaded since start ([#1430](https://github.com/ScilifelabDataCentre/dds_web/pull/1430))
+- New version: 2.3.0 ([#1433](https://github.com/ScilifelabDataCentre/dds_web/pull/1433))
diff --git a/dds_web/__init__.py b/dds_web/__init__.py
@@ -275,7 +275,7 @@ def load_user(user_id):
             set_expired_to_archived,
             delete_invites,
             quarterly_usage,
-            reporting_units_and_users,
+            collect_stats,
             monitor_usage,
         )
 
@@ -290,7 +290,7 @@ def load_user(user_id):
         app.cli.add_command(set_expired_to_archived)
         app.cli.add_command(delete_invites)
         app.cli.add_command(quarterly_usage)
-        app.cli.add_command(reporting_units_and_users)
+        app.cli.add_command(collect_stats)
         app.cli.add_command(monitor_usage)
 
         # Make version available inside jinja templates:

diff --git a/dds_web/commands.py b/dds_web/commands.py
@@ -668,20 +668,31 @@ def quarterly_usage():
         raise
 
 
-@click.command("reporting-units-and-users")
+@click.command("stats")
 @flask.cli.with_appcontext
-def reporting_units_and_users():
+def collect_stats():
     """
     At the start of every month, get number of units and users.
     Should be run on the 1st of each month, at around 00:01.
     """
     # Imports
     # Installed
     import flask_mail
+    from sqlalchemy.sql import func
 
     # Own
     import dds_web.utils
-    from dds_web.database.models import Unit, UnitUser, ResearchUser, SuperAdmin, User, Reporting
+    from dds_web.database.models import (
+        Unit,
+        UnitUser,
+        ResearchUser,
+        SuperAdmin,
+        User,
+        Reporting,
+        Project,
+        ProjectUsers,
+        Version,
+    )
 
     # Get current time
     current_time = dds_web.utils.timestamp(ts_format="%Y-%m-%d")
@@ -695,17 +706,51 @@ def reporting_units_and_users():
 
     # New reporting row - numbers are automatically set
     try:
-        unit_count = Unit.query.count()
-        researchuser_count = ResearchUser.query.count()
-        unituser_count = UnitUser.query.count()
+        # User stats
+        researcher_count = ResearchUser.query.count()
+        unit_personnel_count = UnitUser.query.filter_by(is_admin=False).count()
+        unit_admin_count = UnitUser.query.filter_by(is_admin=True).count()
         superadmin_count = SuperAdmin.query.count()
         total_user_count = User.query.count()
+
+        # Unique project owners
+        project_owner_unique_count: int = (
+            ProjectUsers.query.filter_by(owner=True)
+            .with_entities(ProjectUsers.user_id)
+            .distinct()
+            .count()
+        )
+
+        # Project count
+        total_project_count = Project.query.count()
+        active_project_count = Project.query.filter_by(is_active=True).count()
+        inactive_project_count = Project.query.filter_by(is_active=False).count()
+
+        # Unit count
+        unit_count = Unit.query.count()
+
+        # Amount of data
+        bytes_stored_now: int = sum(proj.size for proj in Project.query.filter_by(is_active=True))
+        tb_stored_now: float = round(bytes_stored_now / 1e12, 2)
+        bytes_uploaded_since_start = db.session.query(
+            func.sum(Version.size_stored).label("sum_bytes")
+        ).first()
+        tb_uploaded_since_start: float = round(int(bytes_uploaded_since_start.sum_bytes) / 1e12, 2)
+
+        # Add to database
         new_reporting_row = Reporting(
             unit_count=unit_count,
-            researchuser_count=researchuser_count,
-            unituser_count=unituser_count,
+            researcher_count=researcher_count,
+            unit_personnel_count=unit_personnel_count,
+            unit_admin_count=unit_admin_count,
             superadmin_count=superadmin_count,
             total_user_count=total_user_count,
+            project_owner_unique_count=project_owner_unique_count,
+            total_project_count=total_project_count,
+            active_project_count=active_project_count,
+            inactive_project_count=inactive_project_count,
+            tb_stored_now=tb_stored_now,
+            tb_uploaded_since_start=tb_uploaded_since_start,
         )
         db.session.add(new_reporting_row)
         db.session.commit()

diff --git a/dds_web/database/models.py b/dds_web/database/models.py
@@ -1079,7 +1079,14 @@ class Reporting(db.Model):
     id = db.Column(db.Integer, primary_key=True, autoincrement=True)
     date = db.Column(db.DateTime(), unique=True, nullable=False, default=datetime.date.today)
     unit_count = db.Column(db.Integer, unique=False, nullable=False)
-    researchuser_count = db.Column(db.Integer, unique=False, nullable=False)
-    unituser_count = db.Column(db.Integer, unique=False, nullable=False)
+    researcher_count = db.Column(db.Integer, unique=False, nullable=False)
+    project_owner_unique_count = db.Column(db.Integer, unique=False, nullable=True)
+    unit_personnel_count = db.Column(db.Integer, unique=False, nullable=False)
+    unit_admin_count = db.Column(db.Integer, unique=False, nullable=True)
     superadmin_count = db.Column(db.Integer, unique=False, nullable=False)
     total_user_count = db.Column(db.Integer, unique=False, nullable=False)
+    total_project_count = db.Column(db.Integer, unique=False, nullable=True)
+    active_project_count = db.Column(db.Integer, unique=False, nullable=True)
+    inactive_project_count = db.Column(db.Integer, unique=False, nullable=True)
+    tb_stored_now = db.Column(db.Float, unique=False, nullable=True)
+    tb_uploaded_since_start = db.Column(db.Float, unique=False, nullable=True)
diff --git a/dds_web/version.py b/dds_web/version.py
@@ -1 +1 @@
-__version__ = "2.2.62"
+__version__ = "2.3.0"
diff --git a/migrations/versions/399801a80e7a_change_researchuser_count_column.py b/migrations/versions/399801a80e7a_change_researchuser_count_column.py
@@ -0,0 +1,40 @@
+"""change_researchuser_count_column
+
+Revision ID: 399801a80e7a
+Revises: edde808b4556
+Create Date: 2023-05-29 09:37:31.007336
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = "399801a80e7a"
+down_revision = "edde808b4556"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column(
+        table_name="reporting",
+        column_name="researchuser_count",
+        nullable=False,
+        new_column_name="researcher_count",
+        type_=sa.Integer(),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column(
+        table_name="reporting",
+        column_name="researcher_count",
+        nullable=False,
+        new_column_name="researchuser_count",
+        type_=sa.Integer(),
+    )
+    # ### end Alembic commands ###
diff --git a/migrations/versions/53bb37a3cd84_add_num_unique_proj_owners.py b/migrations/versions/53bb37a3cd84_add_num_unique_proj_owners.py
@@ -0,0 +1,28 @@
+"""add_num_unique_proj_owners
+
+Revision ID: 53bb37a3cd84
+Revises: 93ec6983ce8d
+Create Date: 2023-05-30 13:24:52.406907
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = "53bb37a3cd84"
+down_revision = "93ec6983ce8d"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("reporting", sa.Column("project_owner_unique_count", sa.Integer(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("reporting", "project_owner_unique_count")
+    # ### end Alembic commands ###
diff --git a/migrations/versions/879b99e7f212_unit_personnel_added.py b/migrations/versions/879b99e7f212_unit_personnel_added.py
@@ -0,0 +1,40 @@
+"""unit_personnel_added
+
+Revision ID: 879b99e7f212
+Revises: b976f6cda95c
+Create Date: 2023-05-29 07:51:05.491352
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = "879b99e7f212"
+down_revision = "b976f6cda95c"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column(
+        table_name="reporting",
+        column_name="unituser_count",
+        nullable=False,
+        new_column_name="unit_personnel_count",
+        type_=sa.Integer(),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column(
+        "reporting",
+        "unit_personnel_count",
+        nullable=False,
+        new_column_name="unituser_count",
+        type_=sa.Integer(),
+    )
+    # ### end Alembic commands ###
diff --git a/migrations/versions/93ec6983ce8d_number_of_inactive_projects_added.py b/migrations/versions/93ec6983ce8d_number_of_inactive_projects_added.py
@@ -0,0 +1,28 @@
+"""number_of_inactive_projects_added
+
+Revision ID: 93ec6983ce8d
+Revises: 399801a80e7a
+Create Date: 2023-05-30 08:47:53.926692
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = "93ec6983ce8d"
+down_revision = "399801a80e7a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("reporting", sa.Column("inactive_project_count", sa.Integer(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("reporting", "inactive_project_count")
+    # ### end Alembic commands ###
diff --git a/migrations/versions/aec752f1e0a5_tb_uploaded_since_start_added.py b/migrations/versions/aec752f1e0a5_tb_uploaded_since_start_added.py
@@ -0,0 +1,28 @@
+"""tb_uploaded_since_start_added
+
+Revision ID: aec752f1e0a5
+Revises: d48ecb4db259
+Create Date: 2023-05-31 14:09:07.327919
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = "aec752f1e0a5"
+down_revision = "d48ecb4db259"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("reporting", sa.Column("tb_uploaded_since_start", sa.Float(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("reporting", "tb_uploaded_since_start")
+    # ### end Alembic commands ###
diff --git a/migrations/versions/c1e908241401_total_number_of_projects_added.py b/migrations/versions/c1e908241401_total_number_of_projects_added.py
@@ -0,0 +1,28 @@
+"""total_number_of_projects_added
+
+Revision ID: c1e908241401
+Revises: 879b99e7f212
+Create Date: 2023-05-29 11:27:10.843043
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = "c1e908241401"
+down_revision = "879b99e7f212"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("reporting", sa.Column("total_project_count", sa.Integer(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("reporting", "total_project_count")
+    # ### end Alembic commands ###
diff --git a/migrations/versions/d48ecb4db259_add_terrabytes_stored_in_system.py b/migrations/versions/d48ecb4db259_add_terrabytes_stored_in_system.py
@@ -0,0 +1,28 @@
+"""add_terrabytes_stored_in_system
+
+Revision ID: d48ecb4db259
+Revises: 53bb37a3cd84
+Create Date: 2023-05-30 13:44:05.232573
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = "d48ecb4db259"
+down_revision = "53bb37a3cd84"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("reporting", sa.Column("tb_stored_now", sa.Float, nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("reporting", "tb_stored_now")
+    # ### end Alembic commands ###