Merge pull request PGScatalog#316 from fyvon/curation_tracker/litsugg…

…est_import_form

Curation tracker 1.1

curation_tracker/litsuggest.py

@@ -1,9 +1,11 @@
-import pandas as pd
+import csv
 import requests
 from pgs_web import constants
 from catalog.models import Publication
 from curation_tracker.models import CurationPublicationAnnotation
 from typing import List
+from io import TextIOWrapper
+from django.core.files.uploadedfile import InMemoryUploadedFile
 
 pgs_db = 'default'
 curation_tracker_db = 'curation_tracker'
@@ -13,12 +15,20 @@ class CurationPublicationAnnotationImport():
     error: str
     skip_reason: str
     annotation: CurationPublicationAnnotation
+    triage_info: dict
 
     def __init__(self, model: CurationPublicationAnnotation = CurationPublicationAnnotation()):
         self.annotation = model# if model else CurationPublicationAnnotation()
         self.error = None
         self.skip_reason = None
-
+        self.triage_info = {}
+
+    def to_dict(self) -> dict:
+        values = annotation_to_dict(self.annotation)
+        values['error'] = self.error
+        values['skip_reason'] = self.skip_reason
+        return values
+
     def is_valid(self) -> bool:
         """Should be used before saving"""
         return self.error == None
@@ -130,18 +140,20 @@ def create_new_annotation(publication_info) -> CurationPublicationAnnotation:
 
     return model
 
-def annotation_to_dict(annotation_import: CurationPublicationAnnotationImport) -> dict:
+def annotation_import_to_dict(annotation_import: CurationPublicationAnnotationImport) -> dict:
     d = dict()
     for attr in ['error','skip_reason']:
         d[attr] = getattr(annotation_import,attr)
+    d['model'] = annotation_to_dict(annotation_import.annotation)
+    return d
+
+def annotation_to_dict(model: CurationPublicationAnnotation) -> dict:
     model_dict = dict()
-    model = annotation_import.annotation
     for attr in ['PMID','study_name','doi','journal','title','year','eligibility','comment',
                  'eligibility_dev_score','eligibility_eval_score','eligibility_description','first_level_curation_status','curation_status',
                  'publication_date']:
         model_dict[attr] = getattr(model,attr)
-    d['model'] = model_dict
-    return d
+    return model_dict
 
 def dict_to_annotation_import(d: dict) -> CurationPublicationAnnotationImport:
     model = CurationPublicationAnnotation()
@@ -171,13 +183,12 @@ def check_study_name(study_name: str) -> str:
         study_name = new_study_name
     return study_name
 
-def litsuggest_import_to_annotation(litsuggest_file: str) -> List[CurationPublicationAnnotationImport]:
-    #df = pd.read_csv(litsuggest_file, sep="\t").dropna(how='all')
-    df = pd.read_csv(litsuggest_file, sep="\t").fillna('') 
+def _litsuggest_IO_to_annotation_imports(litsuggest_file) -> List[CurationPublicationAnnotationImport]:
     models = []
-    for i, row in df.iterrows():
+    reader = csv.DictReader(litsuggest_file, delimiter='\t')
+    for row in reader:
         if not row['pmid']:
-            continue
+            break # litsuggest files might contain a lot of empty rows after the relevant ones
         pmid = str(int(row['pmid']))
         try:
             triage_decision = row['triage.decision']
@@ -193,6 +204,11 @@ def litsuggest_import_to_annotation(litsuggest_file: str) -> List[CurationPublic
             annotationModel.eligibility_description = triage_note
 
             annotation_import = CurationPublicationAnnotationImport(annotationModel)
+            annotation_import.triage_info = {
+                'triage_decision': triage_decision,
+                'triage_note': triage_note,
+                'PMID': pmid
+            }
 
             match triage_decision:
                 case 'New PGS':
@@ -228,4 +244,12 @@ def litsuggest_import_to_annotation(litsuggest_file: str) -> List[CurationPublic
             annotation_import.error = str(e)
             models.append(annotation_import)
 
-    return models
+    return models
+
+def litsuggest_filename_to_annotation_imports(litsuggest_file_name: str) -> List[CurationPublicationAnnotationImport]:
+    with open(litsuggest_file_name, 'r') as litsuggest_file:
+        return _litsuggest_IO_to_annotation_imports(litsuggest_file)
+
+def litsuggest_fileupload_to_annotation_imports(litsuggest_file_upload: InMemoryUploadedFile) -> List[CurationPublicationAnnotationImport]:
+    file_wrapper = TextIOWrapper(litsuggest_file_upload.file)
+    return _litsuggest_IO_to_annotation_imports(file_wrapper)

curation_tracker/migrations/0004_emailtemplate.py

@@ -0,0 +1,31 @@
+# Generated by Django 4.2.6 on 2023-11-27 20:01
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('curation_tracker', '0003_alter_curationpublicationannotation_curation_status'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='EmailTemplate',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('template_type', models.CharField(choices=[('author_data_request', 'Author Data Request')], default='author_data_request', verbose_name='Template Purpose')),
+                ('subject', models.TextField(verbose_name='Email Subject')),
+                ('body', models.TextField(verbose_name='Email Body')),
+                ('is_default', models.BooleanField(default=True, verbose_name='Default Template')),
+                ('created_on', models.DateTimeField(default=django.utils.timezone.now, verbose_name='Created On')),
+                ('last_modified_on', models.DateTimeField(default=django.utils.timezone.now, null=True, verbose_name='Last Modified On')),
+                ('created_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='%(class)s_created_by', to=settings.AUTH_USER_MODEL, verbose_name='Created By')),
+                ('last_modified_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='%(class)s_last_modified_by', to=settings.AUTH_USER_MODEL, verbose_name='Last Modified By')),
+            ],
+        ),
+    ]

curation_tracker/models.py

@@ -3,7 +3,6 @@
 from datetime import date
 from django.db import models
 from pgs_web import constants
-from catalog import common
 from django.utils import timezone
 from django.conf import settings
 # Create your models here.
@@ -206,3 +205,21 @@ def get_epmc_data(self, keep_study_name = False):
             return True
         else:
             return False
+
+
+class EmailTemplate(models.Model):
+    TEMPLATE_TYPES = [
+        ('author_data_request','Author Data Request'),
+    ]
+    template_type = models.CharField('Template Purpose', choices=TEMPLATE_TYPES, null=False, blank=False, default='author_data_request')
+    subject = models.TextField('Email Subject')
+    body = models.TextField('Email Body')
+    is_default = models.BooleanField('Default Template', default=True)
+
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.PROTECT, related_name='%(class)s_created_by', verbose_name='Created By', null=True, blank=True)
+    created_on = models.DateTimeField('Created On', null=False, default=timezone.now)
+    last_modified_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.PROTECT, related_name='%(class)s_last_modified_by', verbose_name='Last Modified By', null=True, blank=True)
+    last_modified_on = models.DateTimeField('Last Modified On', null=True, default=timezone.now)
+
+    def __str__(self): 
+         return f'{self.get_template_type_display()} #{self.id}'

curation_tracker/scripts/import_litsuggest.py

@@ -1,5 +1,5 @@
 import os
-from curation_tracker.litsuggest import litsuggest_import_to_annotation, curation_tracker_db
+from curation_tracker.litsuggest import litsuggest_filename_to_annotation_imports, curation_tracker_db
 
 litsuggest_dir = '/home/florent/PGS_Catalog/Curation/Litsuggest/'
 
@@ -11,7 +11,7 @@ def run():
         if filename.startswith('litsuggest'):
             filepath = f'{litsuggest_dir}{filename}'
             print(f'Reading {filepath}...')
-            annotations = litsuggest_import_to_annotation(filepath)
+            annotations = litsuggest_filename_to_annotation_imports(filepath)
             for annotation in annotations:
                 if not annotation.is_valid():
                     print(f'Study {annotation.annotation.PMID} is not valid: {annotation.error}')

curation_tracker/static/curation_tracker/pgs_admin.css

@@ -11,4 +11,8 @@
 
 .extra-field-button {
     vertical-align: sub;
-}
+}
+
+.litsuggest-import-table * {
+  font-size: 0.7rem;
+}

curation_tracker/static/curation_tracker/pgs_edit_curation_annotation_addon.js

@@ -100,6 +100,27 @@ function autofillForm(){
     _getPublicationInfo({doi: doi, pmid: pmid});
 }
 
+function requestAuthorData(){
+    $.get('../contact-author/', function(data,status){
+        if(status == 'success'){
+            if(data.error){
+                alert(data.error);
+            } else {
+                var email_subject = encodeURIComponent(data.email_subject);
+                var email_body = encodeURIComponent(data.email_body);
+                var cc = encodeURIComponent(data.cc);
+                window.open('mailto:'+'?subject='+email_subject+'&body='+email_body+'&cc='+cc);
+            }
+        } else {
+            alert('Error');
+            console.error(data);
+        }
+    }).fail(function(error){
+        console.error(error);
+        alert('Error: '+error.statusText);
+    })
+}
+
 $(document).ready(function(){
     // Adding 'go to publication' and 'Autofill' buttons after the DOI and PMID form fields 
     $('div.form-row.field-doi.field-PMID > div.flex-container').append('<div><div><a title="Go to the publication page using DOI or the Pubmed page if only the PMID is provided" href="" class="extra-field-button external-link" onclick="goToPublication(); return false;">Go to publication</a></div><div style="display: flex;"><div><a title="Fetch the publication data from EPMC and fill in the form automatically (DOI or PMID required)" href="" class="extra-field-button" onclick="autofillForm(); return false;">Autofill <i class="fa-solid fa-gears"></i></a></div><div id="doi_pmid_error" class="fieldBox errors"><ul class="errorlist"></ul></div></div></div>');

curation_tracker/tables.py

@@ -1,5 +1,4 @@
 import django_tables2 as tables
-from django.conf import settings
 from django.utils.html import format_html
 from .models import *
 

curation_tracker/templates/curation_tracker/litsuggest_preview_form.html

@@ -1,27 +1,32 @@
 {% extends 'admin/base.html' %}
 {% load static %}
 
+{% block extrahead %}{{ block.super }}
+<link rel="stylesheet" href="{% static 'curation_tracker/pgs_admin.css' %}">
+{% endblock %}
+
 {% block content %}
   <div id="content-main">
     <div>
       <h2 style="margin-bottom:1rem">LitSuggest file upload</h2>
+      <form action="./confirm_formset" method="POST" enctype="multipart/form-data">
       <div>
-        <form action="./confirm" method="POST" enctype="multipart/form-data">
-          {{ form.as_p }}
+          {{ comment_form.as_p }}
           <br />
           {% csrf_token %}
           <div class="submit-row">
             <input type="submit" value="Confirm" class="default" style="float:left; font-weight: bold; margin-right: 5px">
             <input class="button" type="button" onclick="history.back()" value="Cancel" />
           </div>
           <div class="help">(Studies with errors and tagged as 'Not PGS' won't be imported)</div>
-        </form>
       </div>
       <div>
         <h2>Import Preview:</h2>
-        <table>
+        <table class="litsuggest-import-table">
           <tr>
             <th>PMID</th>
+            <th>Triage Decision</th>
+            <th>Triage Note</th>
             <th>Study Name</th>
             <th>Title</th>
             <th>Journal</th>
@@ -32,35 +37,48 @@ <h2>Import Preview:</h2>
             <th>Curation Status</th>
             <th>1st Level Curation Status</th>
           </tr>
-          {% for annot in annotations %}
-          {% with model=annot.model %}
+          {{ formset.management_form }}
+          {% for preview_form in formset %}
+          <tr>
+            <td>{{preview_form.triage_info.PMID}}{{preview_form.PMID}}{{preview_form.doi}}{{preview_form.year}}{{preview_form.publication_date}}</td>
+            <td>{{preview_form.triage_info.triage_decision}}</td>
+            <td>{{preview_form.triage_info.triage_note}}</td>
+            <td>{{preview_form.study_name}}{{preview_form.study_name.errors}}</td>
+            <td>{{preview_form.title}}</td>
+            <td>{{preview_form.journal}}</td>
+            <td>{{preview_form.eligibility}}</td>
+            <td>{{preview_form.eligibility_dev_score}}</td>
+            <td>{{preview_form.eligibility_eval_score}}</td>
+            <td>{{preview_form.eligibility_description}}</td>
+            <td>{{preview_form.curation_status}}</td>
+            <td>{{preview_form.first_level_curation_status}}</td>
+          </tr>
+          {% endfor %}
+        </table>
+      </div>
+      <div class="submit-row">
+        <input type="submit" value="Confirm" class="default" style="float:left; font-weight: bold; margin-right: 5px">
+        <input class="button" type="button" onclick="history.back()" value="Cancel" />
+      </div>
+      <div class="help">(Studies with errors and tagged as 'Not PGS' won't be imported)</div>
+      {% if skipped_publications %}
+      <div>
+        <h2>Ignored Publications:</h2>
+        <table>
+          <tr>
+            <th>PMID</th>
+            <th>Comment</th>
+          </tr>
+          {% for skipped_publi in skipped_publications %}
           <tr>
-            <td>{{model.PMID}}</td>
-            {% if annot.error %}
-            <td colspan="100%"><p class="errornote">{{annot.error}}</p></td>
-            {% elif annot.skip_reason %}
-            <td colspan="100%"><b><i>{{annot.skip_reason}}</i></b></td>
-            {% else %}
-            <td>{{model.study_name}}</td>
-            <td>{{model.title}}</td>
-            <td>{{model.journal}}</td>
-            <td>
-            {% if model.eligibility %}
-              <img src="{% static '/admin/img/icon-yes.svg' %}" alt="Yes">
-            {% else %}
-              <img src="{% static '/admin/img/icon-no.svg' %}" alt="No">
-            {% endif %}
-            </td>
-            <td>{{model.eligibility_dev_score}}</td>
-            <td>{{model.eligibility_eval_score}}</td>
-            <td>{{model.eligibility_description}}</td>
-            <td>{{model.curation_status}}</td>
-            <td>{{model.first_level_curation_status}}</td>
-            {% endif %}
+            <td>{{skipped_publi.model.PMID}}</td>
+            <td>{{skipped_publi.skip_reason|default:""}}{{skipped_publi.error|default:""}}</td>
           </tr>
-          {% endwith %}
           {% endfor %}
+        </table>
       </div>
+      {% endif %}
+    </form>
     </div>
   </div>