Merge pull request PGScatalog#395 from fyvon/feature/client_validator

Feature/Pyodide metadata and scores validators

.gcloudignore

@@ -18,6 +18,7 @@
 /catalog/static/
 /rest_api/static/
 /curation_tracker/static/
+/validator/static/
 
 # Curation and release directories (not needed for the website)
 /curation/
@@ -41,6 +42,7 @@
 
 # Virtual env
 /venv/
+.venv
 
 # Other
 /rest_api/fixtures/

.gitignore

@@ -4,4 +4,6 @@
 app.yaml
 pgs-catalog-cred.json
 .idea
-.vscode
+.vscode
+*.min.js
+*.min.css

catalog/static/catalog/pgs.scss

@@ -188,6 +188,12 @@ h2 {
   font-weight:900;
 }
 
+.score_validator_logo:before {
+  font-family: 'Font Awesome 6 Free';
+  content:"\e5a0";
+  font-weight:900;
+}
+
 
 /* Navbar and horizontal menu */
 .navbar {
@@ -2671,4 +2677,13 @@ ul.highlight-first > li:first-child {
   .morelink {
     margin-left: 0 !important;
   }
+}
+
+/* Metadata file validator */
+.custom-file-label {
+  border: 1px solid $pgs_petrol;
+}
+.custom-file-label::after {
+    background-color: $pgs_petrol;
+    color: #FFF;
 }

catalog/templates/catalog/labs.html

@@ -20,16 +20,22 @@ <h2><span class="fas fa-flask pr-2" style="font-size:28px"></span><span>Labs</sp
   </section>
 
   <section>
-    <!-- "Benchmark" button -->
-    <div class="row no-gutters pgs_section mb-5">
-      <div class="col pgs_benchmark" style="justify-content:left">
+    <div class="row no-gutters pgs_section mb-5" style="gap: 40px">
+      <!-- "Benchmark" button -->
+      <div class="pgs_benchmark" style="justify-content:left">
         <a class="pgs_no_icon_link" href="/labs/benchmarking">
           <div>Benchmarking PGS</div>
-          <div>
-            <span class="pgs_benchmark_logo"></span>
-          </div>
+          <div><span class="pgs_benchmark_logo"></span></div>
+        </a>
+      </div>
+      <!-- "Score Validator" button -->
+      <div class="pgs_benchmark" style="justify-content:left">
+        <a class="pgs_no_icon_link" href="{% url 'scoring_files_validation' %}">
+          <div>Score Validator</div>
+          <div><span class="score_validator_logo"></span></div>
         </a>
       </div>
     </div>
   </section>
+
 {% endblock %}

curation_tracker/urls.py

@@ -6,6 +6,6 @@
     path('curation_tracker/l2_curation', views.browse_l2_waiting, name='L2 Curation'),
     path('curation_tracker/release_ready', views.browse_release_ready, name='Release Ready'),
     # e.g. /upload/
-    path("validate_metadata/",  views.validate_metadata_template, name="Metadata Template Validation"),
+    path("validate_metadata_legacy/",  views.validate_metadata_template, name="Metadata Template Validation"),
     path('curation_tracker/stats/', views.stats, name='Curation Stats')
 ]

pgs_web/settings.py

@@ -55,10 +55,11 @@
 # Application definition #
 #------------------------#
 INSTALLED_APPS = [
-	'catalog.apps.CatalogConfig',
+    'catalog.apps.CatalogConfig',
     'rest_api.apps.RestApiConfig',
     'search.apps.SearchConfig',
     'benchmark.apps.BenchmarkConfig',
+    'validator.apps.ValidatorConfig',
     'django.contrib.admin',
     'django.contrib.auth',
     'django.contrib.contenttypes',

pgs_web/urls.py

@@ -18,9 +18,10 @@
 from search import views as search_views
 
 urlpatterns = [
-	path('', include('catalog.urls')),
+    path('', include('catalog.urls')),
     path('', include('rest_api.urls')),
     path('', include('benchmark.urls')),
+    path('', include('validator.urls')),
     re_path(r'^search/', search_views.search, name="PGS Catalog Search"),
     re_path(r'^autocomplete/', search_views.autocomplete, name="PGS Catalog Autocomplete")
 ]

validator/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

validator/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class ValidatorConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'validator'

validator/models.py

@@ -0,0 +1,3 @@
+from django.db import models
+
+# Create your models here.

validator/static/validator/js/metadata_consumer.js

@@ -0,0 +1,123 @@
+const pyworker = await import((on_gae)?'./py-worker.min.js':'./py-worker.js');
+const asyncRun = pyworker.asyncRun;
+
+const validate_metadata = await fetch(new URL('../python/bin/validation_metadata.py', import.meta.url, null)).then(response => response.text());
+let dirHandle;
+
+
+async function validateFile() {
+    const fileInput = document.getElementById('myfile');
+    const file = fileInput.files[0];
+
+    if (file) {
+        const spinner = document.getElementById('pgs_loading');
+        spinner.style.visibility = "visible";
+
+        const reader = new FileReader();
+        reader.onload = async function(event) {
+            const fileContent = new Uint8Array(event.target.result);
+            let context = {
+                file_content: fileContent,
+                file_name: file.name,
+                dirHandle: dirHandle
+            }
+            const { results, error } = await asyncRun(validate_metadata, context);
+            if(results){
+                console.log(results);
+                spinner.style.visibility = "hidden";
+                showResults(results);
+            }
+            if(error){
+                console.error(error);
+                spinner.style.visibility = "hidden";
+                showSystemError(error);
+            }
+        };
+        reader.readAsArrayBuffer(file);
+    }
+}
+
+function report_items_2_html(reports_list) {
+    let report = '<ul>';
+    $.each(reports_list, function(index, report_item){
+      let lines = ''
+      if (report_item.lines) {
+        let lines_label = (report_item.lines.length > 1) ? 'Lines' : 'Line';
+        lines = lines_label+": "+report_item.lines.join(',')+ ' &rarr; ';
+      }
+      let message = report_item.message;
+      // Value highlighting
+      message = message.replace(/"(.+?)"/g, "\"<b>$1</b>\"");
+      // Leading space
+      message = message.replace(/"<b>\s+/g, "\"<b><span class=\"pgs_color_red\">_</span>");
+      // Trailing space
+      message = message.replace(/\s+<\/b>"/g, "<span class=\"pgs_color_red\">_</span></b>\"");
+      // Column highlighting
+      message = message.replace(/'(.+?)'/g, "\'<span class=\"pgs_color_1\">$1</span>\'");
+      report += "<li>"+lines+message+"</li>";
+    });
+    report += '</ul>';
+    return report;
+}
+
+function makeReportTable(data_spreadsheet_items, items_header){
+    let table_html = '<table class="table table-bordered" style="width:auto"><thead class="thead-light">'+
+                     '<tr><th>Spreadsheet</th><th>'+items_header+'</th></tr>'+
+                     '</thead><tbody>';
+        $.each(data_spreadsheet_items, function(spreadsheet, reports_list){
+          table_html += "<tr><td><b>"+spreadsheet+"</b></td><td>";
+          table_html += report_items_2_html(reports_list);
+          table_html += '</td></tr>';
+        });
+        table_html += '</tbody></table>';
+        return table_html;
+}
+
+function showResults(results){
+    let data = JSON.parse(results);
+    let status_style = (data.status === 'failed') ? '<i class="fa fa-times-circle pgs_color_red" style="font-size:18px"></i> Failed' : '<i class="fa fa-check-circle pgs_color_green" style="font-size:18px"></i> Passed';
+    let status_html = '<table class="table table-bordered table_pgs_h mb-4"><tbody>'+
+                        '  <tr><td>File validation</td><td>'+status_style+'</td></tr>'+
+                        '</tbody></table>';
+    $('#check_status').html(status_html);
+    // Error messages
+    if (data.error) {
+        let report = '<h5 class="mt-4"><i class="fa fa-times-circle pgs_color_red"></i> Error report</h5>'
+            + makeReportTable(data.error, 'Error message(s)');
+        $('#report_error').html(report);
+    } else {
+        $('#report_error').html('');
+    }
+    // Warning messages
+    if (data.warning) {
+        let report = '<h5 class="mt-4"><i class="fa fa-exclamation-triangle pgs_color_amber"></i> Warning report</h5>'
+            + makeReportTable(data.warning, 'Warning message(s)');
+        $('#report_warning').html(report);
+    } else {
+        $('#report_warning').html('');
+    }
+    // Other messages
+    if (data.messages){
+        let report = '';
+        $.each(data.messages, function (index, message){
+            report = report + '<div class="alert alert-danger alert-dismissible">'+message+'</div>'+"\n";
+        })
+        $('#report_messages').html(report);
+    }
+}
+
+function showSystemError(errors){
+    let status_html = '<div><b>File validation:</b> <i class="fa fa-times-circle-o pgs_color_red"></i> Failed</div>';
+      $('#check_status').html(status_html);
+      let error_msg = (errors && errors !== '') ? errors : 'Internal error';
+      let error_html = '<div class="clearfix">'+
+                        '  <div class="mt-3 float_left pgs_note pgs_note_2">'+
+                        '    <div><b>Error:</b> '+error_msg+'</div>'+
+                        '  </div>'+
+                        '</div>';
+      $('#report_error').html(error_html);
+}
+
+document.querySelector('#upload_btn').addEventListener('click', async () => {
+    await validateFile();
+});

validator/static/validator/js/py-worker.js

@@ -0,0 +1,32 @@
+// Adapted from https://github.com/EBISPOT/gwas-sumstats-tools-ssf-morph
+// This script is setting up a way to run Python scripts asynchronously in a web worker. It sends the Python script to the worker and sets up a callback to handle the result when the worker has finished executing the script.
+const pyodideWorker = new Worker(new URL((on_gae) ? "webworker.min.js" : "webworker.js", import.meta.url, null));
+
+const callbacks = {};
+
+pyodideWorker.onmessage = (event) => {
+  const { id, ...data } = event.data;
+  const onSuccess = callbacks[id];
+  delete callbacks[id];
+  onSuccess(data);
+};
+//This id is incremented each time the function is invoked and is kept within the safe integer limit.
+
+
+const asyncRun = (() => {
+  let id = 0; // identify a Promise
+  return (script, context) => {
+    // the id could be generated more carefully
+    id = (id + 1) % Number.MAX_SAFE_INTEGER;
+    return new Promise((onSuccess) => {
+      callbacks[id] = onSuccess;
+      pyodideWorker.postMessage({
+        ...context,
+        python: script,
+        id,
+      });
+    });
+  };
+})();
+
+export { asyncRun };