From 499f9bf58c3649714901b151f28f094f59afd26f Mon Sep 17 00:00:00 2001
From: Florent Yvon <florent.yvon76@gmail.com>
Date: Mon, 25 Mar 2024 16:01:29 +0000
Subject: [PATCH 1/7] Added column to catalog_release for counting the EFO
 traits

---
 .../migrations/0003_release_efotrait_count.py  | 18 ++++++++++++++++++
 catalog/models.py                              |  1 +
 2 files changed, 19 insertions(+)
 create mode 100644 catalog/migrations/0003_release_efotrait_count.py

diff --git a/catalog/migrations/0003_release_efotrait_count.py b/catalog/migrations/0003_release_efotrait_count.py
new file mode 100644
index 00000000..c0e275f4
--- /dev/null
+++ b/catalog/migrations/0003_release_efotrait_count.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.2.10 on 2024-03-25 15:57
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('catalog', '0002_alter_metric_name'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='release',
+            name='efotrait_count',
+            field=models.IntegerField(default=0, verbose_name='Number of new EFO traits released'),
+        ),
+    ]
diff --git a/catalog/models.py b/catalog/models.py
index d712796d..bd8525e2 100644
--- a/catalog/models.py
+++ b/catalog/models.py
@@ -1138,6 +1138,7 @@ class Release(models.Model):
     score_count = models.IntegerField('Number of new PGS scores released', default=0)
     performance_count = models.IntegerField('Number of new PGS Performance metrics released', default=0)
     publication_count = models.IntegerField('Number of new PGS Publication released', default=0)
+    efotrait_count = models.IntegerField('Number of new EFO traits released', default=0)
     notes = models.TextField(verbose_name='Release notes', max_length=600, blank=True)
     updated_score_count = models.IntegerField('Number of PGS scores updated', default=0)
     updated_performance_count = models.IntegerField('Number of PGS Performance metrics updated', default=0)

From 0e42587f3680c4a9561bee666f9d9228a10969b0 Mon Sep 17 00:00:00 2001
From: Florent Yvon <florent.yvon76@gmail.com>
Date: Mon, 25 Mar 2024 16:38:59 +0000
Subject: [PATCH 2/7] Added bar chart for released traits

---
 catalog/static/catalog/pgs_bar_charts.js | 2 ++
 catalog/templates/catalog/releases.html  | 6 +++++-
 catalog/views.py                         | 7 ++++++-
 3 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/catalog/static/catalog/pgs_bar_charts.js b/catalog/static/catalog/pgs_bar_charts.js
index 9ab4378a..955ff9a7 100644
--- a/catalog/static/catalog/pgs_bar_charts.js
+++ b/catalog/static/catalog/pgs_bar_charts.js
@@ -4,12 +4,14 @@ bg_colours = {
   'score': '#007C82',
   'publication': '#BE4A81',
   'performance': 'DodgerBlue',
+  'trait': '#c03b23',
   'new_entry': '#FFC200'
 };
 bg_hover_colours = {
   'score': '#00adb5',
   'publication': '#e83e8c',
   'performance': '#51a9ff',
+  'trait': '#f67d51',
   'new_entry': '#FFD700'
 };
 
diff --git a/catalog/templates/catalog/releases.html b/catalog/templates/catalog/releases.html
index 5cc3858b..ce6a83a7 100644
--- a/catalog/templates/catalog/releases.html
+++ b/catalog/templates/catalog/releases.html
@@ -130,6 +130,7 @@ <h3 class="pt-4" style="font-weight:200;border-top:2px dashed #CCC"><i class="fa
         draw_js_barchart(data_chart, 'release_score', 'score');
         draw_js_barchart(data_chart, 'release_publication', 'publication');
         draw_js_barchart(data_chart, 'release_perf', 'performance');
+        draw_js_barchart(data_chart, 'release_trait', 'trait');
         draw_js_barchart_with_label(pub_year_chart.all, 'pub_per_year_all', 'Publications per year (All)', 'publication');
         if (pub_year_chart.nr) {
           draw_js_barchart_with_label(pub_year_chart.r, 'pub_per_year_r', 'Publications per year (Released)', 'publication');
@@ -148,7 +149,7 @@ <h3 class="mt-5 pt-4 mb-4" style="font-weight:200;border-top:2px dashed #CCC"><i
       </div>
     </div>
 
-    <h3 style="font-weight:200"><i class="fa-solid fa-chart-column pgs_color_1 pr-3"></i>Scores, Publications and Performance Metrics per release</h3>
+    <h3 style="font-weight:200"><i class="fa-solid fa-chart-column pgs_color_1 pr-3"></i>Scores, Publications, Performance Metrics and Traits per release</h3>
     <div class="clearfix mb-2">
       <div class="mb-4 mr-4" style="float:left;width:425px;height:300px">
         <canvas id="release_score" width="425" height="300"></canvas>
@@ -159,6 +160,9 @@ <h3 style="font-weight:200"><i class="fa-solid fa-chart-column pgs_color_1 pr-3"
       <div class="mb-4" style="float:left;width:425px;height:300px">
         <canvas id="release_perf" width="425" height="300"></canvas>
       </div>
+      <div class="mb-4" style="float:left;width:425px;height:300px">
+        <canvas id="release_trait" width="425" height="300"></canvas>
+      </div>
     </div>
 
     <h3 style="font-weight:200"><i class="fa-solid fa-chart-column pgs_color_2 pr-3"></i>Distribution of publications per year</h3>
diff --git a/catalog/views.py b/catalog/views.py
index 8b04d792..12c67a00 100644
--- a/catalog/views.py
+++ b/catalog/views.py
@@ -689,6 +689,7 @@ def releases(request):
     total_score = 0
     total_perf = 0
     total_publi = 0
+    total_trait = 0
     max_score = 0
     max_publi = 0
     max_perf = 0
@@ -756,6 +757,7 @@ def releases(request):
         score = release.score_count
         perf  = release.performance_count
         publi = release.publication_count
+        trait = release.efotrait_count
         date  = release.date
 
         release_item = {
@@ -763,13 +765,16 @@ def releases(request):
                         'score_count': score,
                         'performance_count': perf,
                         'publication_count': publi,
+                        'trait_count': trait,
                         'total_score_count': total_score,
                         'total_performance_count': total_perf,
-                        'total_publication_count': total_publi
+                        'total_publication_count': total_publi,
+                        'total_trait_count': total_trait
                        }
         total_score += score
         total_perf += perf
         total_publi += publi
+        total_trait += trait
 
         release_data.append(release_item)
 

From b8d0818da7e4cbdfe90872cbf11a41e62233fcfd Mon Sep 17 00:00:00 2001
From: Florent Yvon <florent.yvon76@gmail.com>
Date: Tue, 26 Mar 2024 11:18:32 +0000
Subject: [PATCH 3/7] Added new traits to releases table

---
 catalog/models.py                        |  6 ++++++
 catalog/static/catalog/pgs_bar_charts.js |  9 +++++++++
 catalog/templates/catalog/releases.html  | 20 ++++++++++++++++++++
 rest_api/serializers.py                  |  4 ++--
 4 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/catalog/models.py b/catalog/models.py
index bd8525e2..32bc47f7 100644
--- a/catalog/models.py
+++ b/catalog/models.py
@@ -1161,3 +1161,9 @@ def released_publication_ids(self):
     def released_performance_ids(self):
         performances = Performance.objects.values_list('id', flat=True).filter(date_released__exact=self.date).order_by('id')
         return list(performances)
+
+    @property
+    def released_new_trait_ids(self):
+        previous_traits = Score.objects.values_list('trait_efo__id', flat=True).filter(date_released__lt=self.date).distinct()
+        new_traits = Score.objects.values_list('trait_efo__id', flat=True).filter(date_released__exact=self.date).distinct()
+        return list(new_traits.difference(previous_traits))
diff --git a/catalog/static/catalog/pgs_bar_charts.js b/catalog/static/catalog/pgs_bar_charts.js
index 955ff9a7..e30e0627 100644
--- a/catalog/static/catalog/pgs_bar_charts.js
+++ b/catalog/static/catalog/pgs_bar_charts.js
@@ -43,6 +43,15 @@ $('.get_pgs_ids').click(function() {
         html_pub += '</ul>';
         $(list_pub_id).html(html_pub);
         $('#list_'+id).show();
+
+        list_trait_id = '#list_'+id.replace(type,'trait');
+        html_trait = "<ul>";
+        $.each(result.released_new_trait_ids, function(index, trait_id) {
+          html_trait += '<li><a href="/trait/'+trait_id+'/">'+trait_id+'</a></li>';
+        });
+        html_trait += '</ul>';
+        $(list_trait_id).html(html_trait);
+        $('#list_'+id).show();
     })
     .fail(function (xhRequest, ErrorText, thrownError) {
       console.log(xhRequest);
diff --git a/catalog/templates/catalog/releases.html b/catalog/templates/catalog/releases.html
index ce6a83a7..680e32a8 100644
--- a/catalog/templates/catalog/releases.html
+++ b/catalog/templates/catalog/releases.html
@@ -20,6 +20,7 @@ <h3 style="font-weight:200"><i class="far fa-calendar-check pgs_color_1 pr-3"></
               <th>Score(s)</th>
               <th>Publication(s)</th>
               <th>Performance(s)</th>
+              <th>Trait(s)</th>
               <th>Notes</th>
             </tr>
           </thead>
@@ -53,6 +54,15 @@ <h3 style="font-weight:200"><i class="far fa-calendar-check pgs_color_1 pr-3"></
                     -
                   {% endif %}
                 </td>
+                <td>
+                  {% if latest_release.efotrait_count > 0 %}
+                    <span class="badge badge-pill badge-pgs-2">{{ latest_release.efotrait_count }}</span>
+                    <a class="toggle_btn pgs_btn_plus get_pgs_ids pgs_no_icon_link ml-1" id="release_trait_{{ latest_release.date|date:'Y-m-d' }}" data-toggle="tooltip" title="Click to display the list of Trait IDs"></a>
+                    <div class="toggle_list mt-2" id="list_release_trait_{{ latest_release.date|date:'Y-m-d' }}"></div>
+                  {% else %}
+                    -
+                  {% endif %}
+                </td>
                 <td style="text-align:left">{{ latest_release.notes }}</td>
               </tr>
           </tbody>
@@ -70,6 +80,7 @@ <h3 class="pt-4" style="font-weight:200;border-top:2px dashed #CCC"><i class="fa
               <th>Score(s)</th>
               <th>Publication(s)</th>
               <th>Performance(s)</th>
+              <th>Trait(s)</th>
               <th>Notes</th>
             </tr>
           </thead>
@@ -109,6 +120,15 @@ <h3 class="pt-4" style="font-weight:200;border-top:2px dashed #CCC"><i class="fa
                     -
                   {% endif %}
                 </td>
+                <td>
+                  {% if latest_release.efotrait_count > 0 %}
+                    <span class="badge badge-pill badge-pgs-2">{{ release.efotrait_count }}</span>
+                    <a class="toggle_btn pgs_btn_plus get_pgs_ids pgs_no_icon_link ml-1" id="release_trait_{{ release.date|date:'Y-m-d' }}" data-toggle="tooltip" title="Click to display the list of Trait IDs"></a>
+                    <div class="toggle_list mt-2" id="list_release_trait_{{ release.date|date:'Y-m-d' }}"></div>
+                  {% else %}
+                    -
+                  {% endif %}
+                </td>
                 <td style="text-align:left">{{ release.notes }}</td>
               </tr>
           {% endfor %}
diff --git a/rest_api/serializers.py b/rest_api/serializers.py
index efbef993..9d011131 100644
--- a/rest_api/serializers.py
+++ b/rest_api/serializers.py
@@ -196,8 +196,8 @@ class ReleaseSerializer(serializers.ModelSerializer):
 
     class Meta:
         model = Release
-        meta_fields = ('date', 'score_count', 'performance_count', 'publication_count', 'notes',
-                    'released_score_ids', 'released_publication_ids', 'released_performance_ids')
+        meta_fields = ('date', 'score_count', 'performance_count', 'publication_count', 'efotrait_count', 'notes',
+                    'released_score_ids', 'released_publication_ids', 'released_performance_ids', 'released_new_trait_ids')
         fields = meta_fields
         read_only_fields = meta_fields
 

From a48c0c577e9f0ecf43cb85763ad2dec6a299e428 Mon Sep 17 00:00:00 2001
From: Florent Yvon <florent.yvon76@gmail.com>
Date: Tue, 26 Mar 2024 11:19:05 +0000
Subject: [PATCH 4/7] Changed rest url to work with non-live versions

---
 catalog/static/catalog/pgs_bar_charts.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalog/static/catalog/pgs_bar_charts.js b/catalog/static/catalog/pgs_bar_charts.js
index e30e0627..a07c0bdd 100644
--- a/catalog/static/catalog/pgs_bar_charts.js
+++ b/catalog/static/catalog/pgs_bar_charts.js
@@ -1,4 +1,4 @@
-var rest_url = 'https://www.pgscatalog.org/rest/release/'
+var rest_url = '/rest/release/'
 
 bg_colours = {
   'score': '#007C82',

From b11f91c36a62b754c95cf65f340cda64566b1632 Mon Sep 17 00:00:00 2001
From: Florent Yvon <florent.yvon76@gmail.com>
Date: Tue, 26 Mar 2024 11:30:48 +0000
Subject: [PATCH 5/7] Align toggle list elements to left

---
 catalog/templates/catalog/releases.html | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/catalog/templates/catalog/releases.html b/catalog/templates/catalog/releases.html
index 680e32a8..df34d598 100644
--- a/catalog/templates/catalog/releases.html
+++ b/catalog/templates/catalog/releases.html
@@ -33,7 +33,7 @@ <h3 style="font-weight:200"><i class="far fa-calendar-check pgs_color_1 pr-3"></
                   {% if latest_release.score_count > 0 %}
                     <span class="badge badge-pill badge-pgs-2">{{ latest_release.score_count }}</span>
                     <a class="toggle_btn pgs_btn_plus get_pgs_ids pgs_no_icon_link ml-1" id="release_score_{{ latest_release.date|date:'Y-m-d' }}" data-toggle="tooltip" title="Click to display the list of PGS Score IDs"></a>
-                    <div class="toggle_list mt-2" id="list_release_score_{{ latest_release.date|date:'Y-m-d' }}"></div>
+                    <div class="toggle_list mt-2" style="text-align:left" id="list_release_score_{{ latest_release.date|date:'Y-m-d' }}"></div>
                   {% else %}
                     -
                   {% endif %}
@@ -42,7 +42,7 @@ <h3 style="font-weight:200"><i class="far fa-calendar-check pgs_color_1 pr-3"></
                   {% if latest_release.publication_count > 0 %}
                     <span class="badge badge-pill badge-pgs-2">{{ latest_release.publication_count }}</span>
                     <a class="toggle_btn pgs_btn_plus get_pgs_ids pgs_no_icon_link ml-1" id="release_pub_{{ latest_release.date|date:'Y-m-d' }}" data-toggle="tooltip" title="Click to display the list of PGS Publication IDs"></a>
-                    <div class="toggle_list mt-2" id="list_release_pub_{{ latest_release.date|date:'Y-m-d' }}"></div>
+                    <div class="toggle_list mt-2" style="text-align:left" id="list_release_pub_{{ latest_release.date|date:'Y-m-d' }}"></div>
                   {% else %}
                     -
                   {% endif %}
@@ -58,7 +58,7 @@ <h3 style="font-weight:200"><i class="far fa-calendar-check pgs_color_1 pr-3"></
                   {% if latest_release.efotrait_count > 0 %}
                     <span class="badge badge-pill badge-pgs-2">{{ latest_release.efotrait_count }}</span>
                     <a class="toggle_btn pgs_btn_plus get_pgs_ids pgs_no_icon_link ml-1" id="release_trait_{{ latest_release.date|date:'Y-m-d' }}" data-toggle="tooltip" title="Click to display the list of Trait IDs"></a>
-                    <div class="toggle_list mt-2" id="list_release_trait_{{ latest_release.date|date:'Y-m-d' }}"></div>
+                    <div class="toggle_list mt-2" style="text-align:left" id="list_release_trait_{{ latest_release.date|date:'Y-m-d' }}"></div>
                   {% else %}
                     -
                   {% endif %}
@@ -99,7 +99,7 @@ <h3 class="pt-4" style="font-weight:200;border-top:2px dashed #CCC"><i class="fa
                   {% if release.score_count > 0 %}
                     <span class="badge badge-pill badge-pgs-2">{{ release.score_count }}</span>
                     <a class="toggle_btn pgs_btn_plus get_pgs_ids pgs_no_icon_link ml-1" id="release_score_{{ release.date|date:'Y-m-d' }}" data-toggle="tooltip" title="Click to display the list of PGS Score IDs"></a>
-                    <div class="toggle_list mt-2" id="list_release_score_{{ release.date|date:'Y-m-d' }}"></div>
+                    <div class="toggle_list mt-2" style="text-align:left" id="list_release_score_{{ release.date|date:'Y-m-d' }}"></div>
                   {% else %}
                     -
                   {% endif %}
@@ -108,7 +108,7 @@ <h3 class="pt-4" style="font-weight:200;border-top:2px dashed #CCC"><i class="fa
                   {% if release.publication_count > 0 %}
                     <span class="badge badge-pill badge-pgs-2">{{ release.publication_count }}</span>
                     <a class="toggle_btn pgs_btn_plus get_pgs_ids pgs_no_icon_link ml-1" id="release_pub_{{ release.date|date:'Y-m-d' }}" data-toggle="tooltip" title="Click to display the list of PGS Publication IDs"></a>
-                    <div class="toggle_list mt-2" id="list_release_pub_{{ release.date|date:'Y-m-d' }}"></div>
+                    <div class="toggle_list mt-2" style="text-align:left" id="list_release_pub_{{ release.date|date:'Y-m-d' }}"></div>
                   {% else %}
                     -
                   {% endif %}
@@ -124,7 +124,7 @@ <h3 class="pt-4" style="font-weight:200;border-top:2px dashed #CCC"><i class="fa
                   {% if latest_release.efotrait_count > 0 %}
                     <span class="badge badge-pill badge-pgs-2">{{ release.efotrait_count }}</span>
                     <a class="toggle_btn pgs_btn_plus get_pgs_ids pgs_no_icon_link ml-1" id="release_trait_{{ release.date|date:'Y-m-d' }}" data-toggle="tooltip" title="Click to display the list of Trait IDs"></a>
-                    <div class="toggle_list mt-2" id="list_release_trait_{{ release.date|date:'Y-m-d' }}"></div>
+                    <div class="toggle_list mt-2" style="text-align:left" id="list_release_trait_{{ release.date|date:'Y-m-d' }}"></div>
                   {% else %}
                     -
                   {% endif %}

From b2ef2eec3403f753016e3f94d888e9a7e9cec397 Mon Sep 17 00:00:00 2001
From: Florent Yvon <florent.yvon76@gmail.com>
Date: Tue, 26 Mar 2024 17:47:31 +0000
Subject: [PATCH 6/7] Updating traits count during release

---
 release/scripts/CreateRelease.py      | 9 ++++++++-
 release/scripts/run_release_script.py | 1 +
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/release/scripts/CreateRelease.py b/release/scripts/CreateRelease.py
index 9b38be81..2f14eb04 100644
--- a/release/scripts/CreateRelease.py
+++ b/release/scripts/CreateRelease.py
@@ -11,6 +11,7 @@ class CreateRelease:
 
     new_scores = {}
     new_performances = {}
+    new_traits = set()
 
     def __init__(self, release_tomorrow=None):
         self.release_tomorrow = release_tomorrow
@@ -30,6 +31,7 @@ def update_data_to_release(self):
         by adding a date in the 'date_released' columns
         """
         self.get_release_date()
+        previous_traits = set(Score.objects.values_list('trait_efo__id', flat=True).exclude(date_released__isnull=True).distinct())
         #### Add release date for each publications and dependent models ####
         for publication in self.new_publications:
             publication.date_released = self.new_release_date
@@ -42,6 +44,9 @@ def update_data_to_release(self):
                 # Update date_release
                 score.date_released = self.new_release_date
                 score.save()
+                # Get new traits
+                score_traits = {efotrait.id for efotrait in score.trait_efo.all()}
+                self.new_traits.update(score_traits.difference(previous_traits))
 
             # Performances
             performances_list = Performance.objects.filter(date_released__isnull=True, publication=publication)
@@ -55,12 +60,13 @@ def update_data_to_release(self):
     def create_new_release(self):
         """ Create new release instance and save it in the database """
         #### Create new release instance ####
-        release_notes = 'This release contains {} new Score(s), {} new Publication(s) and {} new Performance metric(s)'.format(len(self.new_scores.keys()), len(self.new_publications), len(self.new_performances.keys()))
+        release_notes = 'This release contains {} new Score(s), {} new Publication(s), {} new Performance metric(s) and {} new Trait(s)'.format(len(self.new_scores.keys()), len(self.new_publications), len(self.new_performances.keys()), len(self.new_traits))
         release = Release.objects.create(
                 date=self.new_release_date,
                 performance_count=len(self.new_performances.keys()),
                 publication_count=len(self.new_publications),
                 score_count=len(self.new_scores.keys()),
+                efotrait_count=len(self.new_traits),
                 notes=release_notes
               )
         return release
@@ -123,6 +129,7 @@ def run():
     print(', '.join(release.new_scores.keys()))
     print("Number of new Publications: "+str(new_release.publication_count))
     print("Number of new Performances: "+str(new_release.performance_count))
+    print("Number of new Traits: " + str(new_release.efotrait_count))
 
     # Scores
     scores_direct = Score.objects.filter(date_released__isnull=True)
diff --git a/release/scripts/run_release_script.py b/release/scripts/run_release_script.py
index 266c281f..338bb393 100644
--- a/release/scripts/run_release_script.py
+++ b/release/scripts/run_release_script.py
@@ -181,6 +181,7 @@ def call_create_release():
     output_report(', '.join(release.new_scores.keys()))
     output_report("Number of new Publications: "+str(new_release.publication_count))
     output_report("Number of new Performances: "+str(new_release.performance_count))
+    output_report("Number of new Traits: " + str(new_release.efotrait_count))
 
     if new_release.score_count == 0 or new_release.publication_count == 0 or new_release.performance_count == 0:
         error_report("at least one of the main components (Score, Publication or Performance Metrics) hasn't a new entry this release")

From ee849c95eb3ad4bbc62fb266fddcc294223b3b35 Mon Sep 17 00:00:00 2001
From: Florent Yvon <florent.yvon76@gmail.com>
Date: Tue, 2 Apr 2024 15:48:06 +0100
Subject: [PATCH 7/7] Fixed efo trait count test using latest release instead
 of each older release

---
 catalog/templates/catalog/releases.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/catalog/templates/catalog/releases.html b/catalog/templates/catalog/releases.html
index df34d598..8aefa8ce 100644
--- a/catalog/templates/catalog/releases.html
+++ b/catalog/templates/catalog/releases.html
@@ -121,7 +121,7 @@ <h3 class="pt-4" style="font-weight:200;border-top:2px dashed #CCC"><i class="fa
                   {% endif %}
                 </td>
                 <td>
-                  {% if latest_release.efotrait_count > 0 %}
+                  {% if release.efotrait_count > 0 %}
                     <span class="badge badge-pill badge-pgs-2">{{ release.efotrait_count }}</span>
                     <a class="toggle_btn pgs_btn_plus get_pgs_ids pgs_no_icon_link ml-1" id="release_trait_{{ release.date|date:'Y-m-d' }}" data-toggle="tooltip" title="Click to display the list of Trait IDs"></a>
                     <div class="toggle_list mt-2" style="text-align:left" id="list_release_trait_{{ release.date|date:'Y-m-d' }}"></div>