Merge pull request #530 from hubmapconsortium/Derek-Furst/track-status

Derek furst/track status

src/app.py

@@ -1294,6 +1294,11 @@ def update_entity(id):
         normalized_status = schema_manager.normalize_status(json_data_dict["status"])
         json_data_dict["status"] = normalized_status
 
+    has_updated_status = False
+    if ('status' in json_data_dict) and (json_data_dict['status']):
+        has_updated_status = True
+
+
     # Normalize user provided status
     if "sub_status" in json_data_dict:
         normalized_status = schema_manager.normalize_status(json_data_dict["sub_status"])
@@ -1368,7 +1373,7 @@ def update_entity(id):
         merged_updated_dict = update_entity_details(request, normalized_entity_type, user_token, json_data_dict, entity_dict)
 
         # Handle linkages update via `after_update_trigger` methods
-        if has_direct_ancestor_uuids or has_associated_collection_uuid:
+        if has_direct_ancestor_uuids or has_associated_collection_uuid or has_updated_status:
             after_update(normalized_entity_type, user_token, merged_updated_dict)
     elif normalized_entity_type == 'Upload':
         has_dataset_uuids_to_link = False
@@ -1400,7 +1405,7 @@ def update_entity(id):
         merged_updated_dict = update_entity_details(request, normalized_entity_type, user_token, json_data_dict, entity_dict)
 
         # Handle linkages update via `after_update_trigger` methods
-        if has_dataset_uuids_to_link or has_dataset_uuids_to_unlink:
+        if has_dataset_uuids_to_link or has_dataset_uuids_to_unlink or has_updated_status:
             after_update(normalized_entity_type, user_token, merged_updated_dict)
     elif normalized_entity_type == 'Collection':
         # Generate 'before_update_trigger' data and update the entity details in Neo4j

src/schema/provenance_schema.yaml

@@ -304,9 +304,12 @@ ENTITIES:
         before_property_update_validators: 
           - validate_application_header_before_property_update
           - validate_dataset_status_value
+          - validate_status_changed
         generated: true
         description: "One of: New|Processing|QA|Published|Error|Hold|Invalid|Submitted"
         before_create_trigger: set_dataset_status_new
+        after_create_trigger: set_status_history
+        after_update_trigger: set_status_history
       title:
         type: string
         generated: true # Disallow entry from users via POST
@@ -475,6 +478,11 @@ ENTITIES:
       dbgap_study_url:
         type: string
         description: 'A URL linking the dataset to the particular study on dbGap it belongs to'
+      status_history:
+        type: list
+        description: "A list of all status change events. Each entry in the list is a dictionary containing the change_timestamp, changed_by_email, previous_status, new_status"
+        generated: true
+        immutable: true
 
   ############################################# Publication #############################################
   Publication:
@@ -518,9 +526,12 @@ ENTITIES:
         before_property_update_validators:
           - validate_application_header_before_property_update
           - validate_dataset_status_value
+          - validate_status_changed
         generated: true
         description: "One of: New|Processing|QA|Published|Error|Hold|Invalid"
         before_create_trigger: set_dataset_status_new
+        after_create_trigger: set_status_history
+        after_update_trigger: set_status_history
       title:
         type: string
         description: "The title of the publication."
@@ -737,6 +748,11 @@ ENTITIES:
         description: "The uuid of the associated collection for a given publication"
         after_create_trigger: link_publication_to_associated_collection
         after_update_trigger: link_publication_to_associated_collection
+      status_history:
+        type: list
+        description: "A list of all status change events. Each entry in the list is a dictionary containing the change_timestamp, changed_by_email, previous_status, new_status"
+        generated: true
+        immutable: true
 
   ############################################# Donor #############################################
   Donor:
@@ -1112,11 +1128,14 @@ ENTITIES:
         before_property_update_validators: 
           - validate_application_header_before_property_update
           - validate_upload_status_value
+          - validate_status_changed
         type: string
         generated: true
         description: "One of: New|Valid|Invalid|Error|Reorganized|Processing"
         # Trigger method will set the status to "New" on create
         before_create_trigger: set_upload_status_new
+        after_create_trigger: set_status_history
+        after_update_trigger: set_status_history
       validation_message:
         type: string
         description: A message from the validation tools describing what is invalid with the upload.
@@ -1164,3 +1183,8 @@ ENTITIES:
         description: "The datasets that are contained in this Upload."
         # A few time-consuming properties (with read triggers) of each dataset are excluded
         on_read_trigger: get_upload_datasets
+      status_history:
+        type: list
+        description: "A list of all status change events. Each entry in the list is a dictionary containing the change_timestamp, changed_by_email, previous_status, new_status"
+        generated: true
+        immutable: true

src/schema/schema_triggers.py

@@ -556,6 +556,41 @@ def update_file_descriptions(property_key, normalized_type, user_token, existing
 
 
 
+####################################################################################################
+## Trigger methods shared by Dataset, Upload, and Publication - DO NOT RENAME
+####################################################################################################
+
+def set_status_history(property_key, normalized_type, user_token, existing_data_dict, new_data_dict):
+    new_status_history = []
+    status_entry = {}
+
+    if 'status_history' in existing_data_dict:
+        status_history_string = existing_data_dict['status_history'].replace("'", "\"")
+        new_status_history += json.loads(status_history_string)
+
+    if 'status' not in existing_data_dict:
+        raise KeyError("Missing 'status' key in 'existing_data_dict' during calling 'set_status_history()' trigger method")
+    if 'last_modified_timestamp' not in existing_data_dict:
+        raise KeyError("Missing 'last_modified_timestamp' key in 'existing_dat_dict' during calling 'set_status_history()' trigger method.")
+    if 'last_modified_user_email' not in existing_data_dict:
+        raise KeyError("Missing 'last_modified_user_email' key in 'existing_data_dict' during calling 'set_status_hisotry()' trigger method.")
+
+    status = existing_data_dict['status']
+    last_modified_user_email = existing_data_dict['last_modified_user_email']
+    last_modified_timestamp = existing_data_dict['last_modified_timestamp']
+    uuid = existing_data_dict['uuid']
+
+    status_entry['status'] = status
+    status_entry['changed_by_email'] = last_modified_user_email
+    status_entry['change_timestamp'] = last_modified_timestamp
+    new_status_history.append(status_entry)
+    entity_data_dict = {"status_history": new_status_history}
+
+    schema_neo4j_queries.update_entity(schema_manager.get_neo4j_driver_instance(), normalized_type, entity_data_dict, uuid)
+
+
+
+
 ####################################################################################################
 ## Trigger methods specific to Collection - DO NOT RENAME
 ####################################################################################################

src/schema/schema_validators.py

@@ -271,6 +271,30 @@ def validate_dataset_status_value(property_key, normalized_entity_type, request,
         raise ValueError(f"Dataset status change to 'Published' can only be made via {SchemaConstants.INGEST_API_APP}")
 
 
+"""
+Validate that status, if included in new_data_dict, is different from the existing status value
+
+Parameters
+----------
+property_key : str
+    The target property key
+normalized_type : str
+    Submission
+request: Flask request object
+    The instance of Flask request passed in from application request
+existing_data_dict : dict
+    A dictionary that contains all existing entity properties
+new_data_dict : dict
+    The json data in request body, already after the regular validations
+"""
+def validate_status_changed(property_key, normalized_entity_type, request, existing_data_dict, new_data_dict):
+    if 'status' not in existing_data_dict:
+        raise KeyError("Missing 'status' key in 'existing_data_dict' during calling 'validate_status_changed()' validator method.")
+    # Only allow 'status' in new_data_dict if its different than the existing status value
+    if existing_data_dict['status'].lower() == new_data_dict['status'].lower():
+        raise ValueError(f"Status value is already {existing_data_dict['status']}, cannot change to {existing_data_dict['status']}. If no change, do not include status field in update")
+
+
 """
 Validate the sub_status field is also provided when Dataset.retraction_reason is provided on update via PUT