Merge branch 'main' into task/DES-2623--pydantic-schemas-for-entities

DesignSafe-CI · Dec 29, 2023 · 17a404f · 17a404f
2 parents f1e4c63 + 9793013
commit 17a404f
Show file tree

Hide file tree

Showing 18 changed files with 107 additions and 71 deletions.
diff --git a/designsafe/apps/api/datafiles/operations/agave_operations.py b/designsafe/apps/api/datafiles/operations/agave_operations.py
@@ -251,12 +251,12 @@ def move(client, src_system, src_path, dest_system, dest_path):
         filePath=urllib.parse.quote(src_path),
         body=body
     )
-    update_meta(
-        src_system=src_system,
-        src_path=src_path,
-        dest_system=dest_system,
-        dest_path=full_dest_path
-    )
+    update_meta.apply_async(kwargs={
+        "src_system": src_system,
+        "src_path": src_path,
+        "dest_system": dest_system,
+        "dest_path": full_dest_path
+    }, queue="indexing")
 
     if os.path.dirname(src_path) != full_dest_path or src_path != full_dest_path:
         agave_indexer.apply_async(kwargs={
@@ -328,12 +328,12 @@ def copy(client, src_system, src_path, dest_system, dest_path):
             urlToIngest=src_url
         )
 
-    copy_meta(
-        src_system=src_system,
-        src_path=src_path,
-        dest_system=dest_system,
-        dest_path=full_dest_path
-    )
+    copy_meta.apply_async(kwargs={
+        "src_system": src_system,
+        "src_path": src_path,
+        "dest_system": dest_system,
+        "dest_path": full_dest_path
+    }, queue='indexing')
 
     if copy_result['nativeFormat'] == 'dir':
         agave_indexer.apply_async(kwargs={
@@ -390,12 +390,12 @@ def rename(client, system, path, new_name):
         filePath=urllib.parse.quote(os.path.join('/', path)),
         body=body
     )
-    update_meta(
-        src_system=system,
-        src_path=path,
-        dest_system=system,
-        dest_path=os.path.join(os.path.dirname(path), new_name)
-    )
+    update_meta.apply_async(kwargs={
+        "src_system": system,
+        "src_path": path,
+        "dest_system": system,
+        "dest_path": os.path.join(os.path.dirname(path), new_name)
+    }, queue="indexing")
 
     # if rename_result['nativeFormat'] == 'dir':
     if listing[0].type == 'dir':

diff --git a/designsafe/apps/api/datafiles/utils.py b/designsafe/apps/api/datafiles/utils.py
@@ -3,8 +3,9 @@
 import logging
 import os
 import re
-import copy
+from copy import deepcopy
 import exifread
+from celery import shared_task
 from designsafe.apps.api.agave import service_account
 from PIL import Image
 
@@ -94,7 +95,15 @@ def scrape_metadata_from_file(file):
 
     return metadata
 
+def create_or_update_meta(client, meta_body):
+    existing_meta = query_file_meta(meta_body["value"]["system"], meta_body["value"]["path"])
+    if not existing_meta:
+        return client.meta.addMetadata(body=json.dumps(meta_body))
+    existing_uuid = existing_meta[0]["uuid"]
+    return client.meta.updateMetadata(uuid=existing_uuid, body=meta_body)
 
+
+@shared_task(autoretry_for=(Exception,), retry_kwargs={'max_retries': 3})
 def update_meta(src_system, src_path, dest_system, dest_path):
     """
     Check for and update metadata record(s)
@@ -105,16 +114,24 @@ def update_meta(src_system, src_path, dest_system, dest_path):
     meta_listing = query_file_meta(system=src_system, path=src_path)
     if meta_listing:
         for meta in meta_listing:
-            meta.value['system'] = dest_system
-            meta.value['path'] = meta.value['path'].replace(src_path, dest_path)
-            meta.value['basePath'] = meta.value['basePath'].replace(
-                os.path.dirname(src_path),
-                os.path.dirname(dest_path)
-            )
-            updates.append({"uuid": meta.uuid, "update": meta})
-        sa_client.meta.bulkUpdate(body=json.dumps(updates))
-
-
+            meta_dict = deepcopy(dict(meta))
+            new_path = meta_dict['value']['path'].replace(src_path.rstrip("/"), dest_path, 1)
+            if not new_path.startswith("/"):
+                new_path = f"/{new_path}"
+
+            new_meta_value = {**meta_dict["value"],
+                                   "system": dest_system,
+                                   "path": new_path,
+                                   "basePath": os.path.dirname(new_path)}
+
+            meta_body = {"name": "designsafe.file", "value": new_meta_value}
+            updates.append({"uuid": meta_dict["uuid"], "body": meta_body})
+
+        for update in updates:
+            sa_client.meta.updateMetadata(uuid=update["uuid"], body=json.dumps(update["body"]))
+
+
+@shared_task(autoretry_for=(Exception,), retry_kwargs={'max_retries': 3})
 def copy_meta(src_system, src_path, dest_system, dest_path):
     """
     Check for and copy metadata record(s)
@@ -125,14 +142,21 @@ def copy_meta(src_system, src_path, dest_system, dest_path):
     meta_listing = query_file_meta(system=src_system, path=src_path)
     if meta_listing:
         for meta in meta_listing:
-            meta_dict = copy.deepcopy(dict(meta))
-            meta_copy = file_meta_obj(
-                path=meta_dict['value']['path'].replace(src_path, dest_path),
-                system=dest_system,
-                meta=meta_dict['value']
-            )
-            copies.append(meta_copy)
-        sa_client.meta.bulkCreate(body=json.dumps(copies))
+            meta_dict = deepcopy(dict(meta))
+            new_path = meta_dict['value']['path'].replace(src_path.rstrip("/"), dest_path, 1)
+            if not new_path.startswith("/"):
+                new_path = f"/{new_path}"
+
+            copy_meta_value = {**meta_dict["value"],
+                                   "system": dest_system,
+                                   "path": new_path,
+                                   "basePath": os.path.dirname(new_path)}
+
+            meta_body = {"name": "designsafe.file", "value": copy_meta_value}
+            copies.append(meta_body)
+
+        for copy in copies:
+            create_or_update_meta(sa_client, copy)
 
 
 def create_meta(path, system, meta):

diff --git a/designsafe/apps/projects/models/agave/base.py b/designsafe/apps/projects/models/agave/base.py
@@ -517,10 +517,12 @@ def to_datacite_json(self,project=None):
         )
         attributes['fundingReferences'] = []
         for award in awards:
-            attributes['fundingReferences'].append({
-                'awardTitle': award['name'],
-                'awardNumber': award['number']
-                })
+            if award.get("number", "").lower().startswith("nsf"):
+                attributes['fundingReferences'].append({
+                    'awardTitle': award['name'],
+                    'awardNumber': award['number'],
+                    "funderName": "National Science Foundation",
+                    })
 
         attributes['relatedIdentifiers'] = []
 

diff --git a/designsafe/apps/projects/models/agave/experimental.py b/designsafe/apps/projects/models/agave/experimental.py
@@ -98,10 +98,12 @@ def to_datacite_json(self, project=None):
             )
             attributes['fundingReferences'] = []
             for award in awards:
-                attributes['fundingReferences'].append({
-                    'awardTitle': award['name'],
-                    'awardNumber': award['number']
-                    })
+                if award.get("number", "").lower().startswith("nsf"):
+                    attributes['fundingReferences'].append({
+                        'awardTitle': award['name'],
+                        'awardNumber': award['number'],
+                        "funderName": "National Science Foundation",
+                        })
         attributes['types']['resourceType'] = "Experiment/{experiment_type}".format(
             experiment_type=self.experiment_type.title()
         )

diff --git a/designsafe/apps/projects/models/agave/hybrid_simulation.py b/designsafe/apps/projects/models/agave/hybrid_simulation.py
@@ -117,10 +117,12 @@ def to_datacite_json(self, project=None):
             )
             attributes['fundingReferences'] = []
             for award in awards:
-                attributes['fundingReferences'].append({
-                    'awardTitle': award['name'],
-                    'awardNumber': award['number']
-                    })
+                if award.get("number", "").lower().startswith("nsf"):
+                    attributes['fundingReferences'].append({
+                        'awardTitle': award['name'],
+                        'awardNumber': award['number'],
+                        "funderName": "National Science Foundation",
+                        })
         # related works are not required, so they can be missing...
         attributes['relatedIdentifiers'] = []
         for r_work in self.related_work:

diff --git a/designsafe/apps/projects/models/agave/rapid.py b/designsafe/apps/projects/models/agave/rapid.py
@@ -115,10 +115,12 @@ def to_datacite_json(self, project=None):
             )
             attributes['fundingReferences'] = []
             for award in awards:
-                attributes['fundingReferences'].append({
-                    'awardTitle': award['name'],
-                    'awardNumber': award['number']
-                    })
+                if award.get("number", "").lower().startswith("nsf"):
+                    attributes['fundingReferences'].append({
+                        'awardTitle': award['name'],
+                        'awardNumber': award['number'],
+                        "funderName": "National Science Foundation",
+                        })
         # related works are not required, so they can be missing...
         attributes['relatedIdentifiers'] = []
         for r_work in self.related_work:
@@ -303,10 +305,12 @@ def to_datacite_json(self, project=None):
             )
             attributes['fundingReferences'] = []
             for award in awards:
-                attributes['fundingReferences'].append({
-                    'awardTitle': award['name'],
-                    'awardNumber': award['number']
-                    })
+                if award.get("number", "").lower().startswith("nsf"):
+                    attributes['fundingReferences'].append({
+                        'awardTitle': award['name'],
+                        'awardNumber': award['number'],
+                        "funderName": "National Science Foundation",
+                        })
         # related works are not required, so they can be missing...
         attributes['relatedIdentifiers'] = []
         for r_work in self.related_work:

diff --git a/designsafe/apps/projects/models/agave/simulation.py b/designsafe/apps/projects/models/agave/simulation.py
@@ -101,10 +101,12 @@ def to_datacite_json(self, project=None):
             )
             attributes['fundingReferences'] = []
             for award in awards:
-                attributes['fundingReferences'].append({
-                    'awardTitle': award['name'],
-                    'awardNumber': award['number']
-                    })
+                if award.get("number", "").lower().startswith("nsf"):
+                    attributes['fundingReferences'].append({
+                        'awardTitle': award['name'],
+                        'awardNumber': award['number'],
+                        "funderName": "National Science Foundation",
+                        })
         # related works are not required, so they can be missing...
         attributes['relatedIdentifiers'] = []
         for r_work in self.related_work:

diff --git a/...gnsafe/components/modals/data-browser-service-copy/data-browser-service-copy.component.js b/...gnsafe/components/modals/data-browser-service-copy/data-browser-service-copy.component.js
@@ -36,7 +36,6 @@ class DataBrowserCopyCtrl {
         this.options = [
             { api: 'agave', label: 'My Data' },
             { api: 'projects', label: 'My Projects' },
-            { api: 'googledrive', label: 'Google Drive' },
             { api: 'box', label: 'Box' },
             { api: 'dropbox', label: 'Dropbox' },
         ];

diff --git a/...rojects/components/manage-field-recon/documents/manage-field-recon-documents.component.js b/...rojects/components/manage-field-recon/documents/manage-field-recon-documents.component.js
@@ -203,7 +203,7 @@ class ManageFieldReconDocumentsCtrl {
         if (!facilityId) {
             delete this.form.facility;
         } else if (facilityId === 'other') {
-            this.form.facility = { id: 'other', name: this.form.facility.name };
+            this.form.facility = { id: 'other', name: this.form.facility.name ?? ''};
         } else {
             this.form.facility = {
                 id: facilityId,

diff --git a/...ojects/components/manage-field-recon/documents/manage-field-recon-documents.template.html b/...ojects/components/manage-field-recon/documents/manage-field-recon-documents.template.html
@@ -44,7 +44,7 @@ <h4 class="modal-title" style="border:none;">
                         ng-model="$ctrl.form.facility.id">
                         <option value="">-- Select a Facility --</option>
                     </select>
-                    <input class="form-control" type="text" placeholder="Custom Facility" style="margin-left: 5px"
+                    <input class="form-control" type="text" placeholder="Custom Facility" ng-required="true" style="margin-left: 5px"
                         ng-model="$ctrl.form.facility.name" ng-if="$ctrl.form.facility.id == 'other'" />
                 </div>
             </div>

diff --git a/.../projects/components/manage-field-recon/missions/manage-field-recon-missions.component.js b/.../projects/components/manage-field-recon/missions/manage-field-recon-missions.component.js
@@ -204,7 +204,7 @@ class ManageFieldReconMissionsCtrl {
         if (!facilityId) {
             delete this.form.facility;
         } else if (facilityId === 'other') {
-            this.form.facility = { id: 'other', name: this.form.facility.name };
+            this.form.facility = { id: 'other', name: this.form.facility.name ?? ''};
         } else {
             this.form.facility = {
                 id: facilityId,

diff --git a/...projects/components/manage-field-recon/missions/manage-field-recon-missions.template.html b/...projects/components/manage-field-recon/missions/manage-field-recon-missions.template.html
@@ -85,7 +85,7 @@ <h4 class="modal-title" style="border:none;">
                         style="flex: 1; min-width: 150px">
                         <option value="">-- Select a Facility --</option>
                     </select>
-                    <input class="form-control" type="text" placeholder="Custom Facility" ng-model="$ctrl.form.facility.name"
+                    <input class="form-control" type="text" placeholder="Custom Facility" ng-required="true" ng-model="$ctrl.form.facility.name"
                         ng-if="$ctrl.form.facility.id == 'other'" style="margin-left: 5px;" />
                 </div>
             </div>

diff --git a/...ipts/projects/components/manage-hybrid-simulations/manage-hybrid-simulations.component.js b/...ipts/projects/components/manage-hybrid-simulations/manage-hybrid-simulations.component.js
@@ -207,7 +207,7 @@ class ManageHybridSimCtrl {
         if (!facilityId) {
             delete this.form.facility;
         } else if (facilityId === 'other') {
-            this.form.facility = { id: 'other', name: this.form.facility.name };
+            this.form.facility = { id: 'other', name: this.form.facility.name ?? ''};
         } else {
             this.form.facility = {
                 id: facilityId,

diff --git a/...pts/projects/components/manage-hybrid-simulations/manage-hybrid-simulations.template.html b/...pts/projects/components/manage-hybrid-simulations/manage-hybrid-simulations.template.html
@@ -74,7 +74,7 @@ <h4 class="modal-title" style="border:none;">
                         style="flex: 1; min-width: 150px">
                         <option value="">-- Select a Facility --</option>
                     </select>
-                    <input class="form-control" type="text" placeholder="Custom Facility" ng-model="$ctrl.form.facility.name"
+                    <input class="form-control" type="text" placeholder="Custom Facility" ng-required="true" ng-model="$ctrl.form.facility.name"
                         ng-if="$ctrl.form.facility.id == 'other'" style="margin-left: 5px;" />
                 </div>
             </div>

diff --git a/designsafe/static/scripts/projects/components/manage-project/manage-project.component.js b/designsafe/static/scripts/projects/components/manage-project/manage-project.component.js
@@ -225,7 +225,7 @@ class ManageProjectCtrl {
 
                 const facilities = this.form.facilities.filter((fac) => fac && !!fac.id).map((fac) => {
                     if (fac.id === 'other') {
-                        return {id: 'other', name: fac.name}
+                        return {id: 'other', name: fac.name ?? ""}
                     } else {
                         return {id: fac.id, name: this.ui.facilities.find((f) => f.name === fac.id).label}
                     }

diff --git a/designsafe/static/scripts/projects/components/manage-project/manage-project.template.html b/designsafe/static/scripts/projects/components/manage-project/manage-project.template.html
@@ -280,6 +280,7 @@ <h3 class="text-center">
                     type="text"
                     style="margin-left: 5px"
                     placeholder="Custom Facility"
+                    ng-required="true"
                     ng-model="$ctrl.form.facilities[$index].name"
                     ng-if="$ctrl.form.facilities[$index].id == 'other'"/>
                 </div>

diff --git a/...afe/static/scripts/projects/components/manage-simulations/manage-simulations.component.js b/...afe/static/scripts/projects/components/manage-simulations/manage-simulations.component.js
@@ -198,7 +198,7 @@ class ManageSimulationCtrl {
         if (!facilityId) {
             delete this.form.facility;
         } else if (facilityId === 'other') {
-            this.form.facility = { id: 'other', name: this.form.facility.name };
+            this.form.facility = { id: 'other', name: this.form.facility.name ?? ''};
         } else {
             this.form.facility = {
                 id: facilityId,

diff --git a/...fe/static/scripts/projects/components/manage-simulations/manage-simulations.template.html b/...fe/static/scripts/projects/components/manage-simulations/manage-simulations.template.html
@@ -85,7 +85,7 @@ <h4 class="modal-title" style="border:none;">
                         ng-model="$ctrl.form.facility.id">
                         <option value="">-- Select a Facility --</option>
                     </select>
-                    <input class="form-control" type="text" placeholder="Custom Facility" style="margin-left: 5px"
+                    <input class="form-control" ng-required="true" type="text" placeholder="Custom Facility" style="margin-left: 5px"
                         ng-model="$ctrl.form.facility.name" ng-if="$ctrl.form.facility.id == 'other'" />
                 </div>
             </div>