Merge pull request #962 from Sage-Bionetworks/develop

Schematic Release V22.10.2

api/openapi/api.yaml

@@ -849,3 +849,100 @@ paths:
           description: Check schematic log.
       tags:
         - Schema Operation
+  /visualize/tangled_tree/layers:
+    get:
+      summary: Get layers of tangled tree.
+      description: >-
+        Get tangled tree node layers to display for a given data model and figure type
+      operationId: api.routes.get_viz_tangled_tree_layers
+      parameters:
+        - in: query
+          name: schema_url
+          schema:
+            type: string
+          description: Data Model URL
+          example: >-
+            https://raw.githubusercontent.com/Sage-Bionetworks/schematic/develop/tests/data/example.model.jsonld
+          required: true
+        - in: query
+          name: figure_type
+          schema:
+            type: string
+            enum: ["component", "dependency"]
+          description: Figure type to generate.
+          example: 'component'
+          required: true
+      responses:
+        "200":
+          description: Returns a dataframe as a JSON String.
+          content:
+            text/json:
+              schema:
+                type: string
+      tags:
+        - Visualization Operations
+  /visualize/tangled_tree/text:
+    get:
+      summary: Get text to display on tangled tree.
+      description: >-
+        Get tangled tree plain or higlighted text to display for a given data model, text formatting and figure type
+      operationId: api.routes.get_viz_tangled_tree_text
+      parameters:
+        - in: query
+          name: schema_url
+          schema:
+            type: string
+          description: Data Model URL
+          example: >-
+            https://raw.githubusercontent.com/Sage-Bionetworks/schematic/develop/tests/data/example.model.jsonld
+          required: true
+        - in: query
+          name: figure_type
+          schema:
+            type: string
+            enum: ["component", "dependency"]
+          description: Figure type to generate.
+          example: 'component'
+          required: true
+        - in: query
+          name: text_format
+          schema:
+            type: string
+            enum: ["plain", "highlighted"]
+          description: Text formatting type.
+          example: 'plain'
+          required: true
+      responses:
+        "200":
+          description: Returns a dataframe as a JSON String.
+          content:
+            text/csv:
+              schema:
+                type: string
+      tags:
+        - Visualization Operations
+  /visualize/attributes:
+    get:
+      summary: Get an attributes table for a data model, as a CSV (JSON String)
+      description: >-
+        Get all the attributes associated with a data model formatted as a
+        dataframe (stored as a JSON String) for use in Observable visualization.
+      operationId: api.routes.get_viz_attributes_explorer
+      parameters:
+        - in: query
+          name: schema_url
+          schema:
+            type: string
+          description: Data Model URL
+          example: >-
+            https://raw.githubusercontent.com/Sage-Bionetworks/schematic/develop/tests/data/example.model.jsonld
+          required: true
+      responses:
+        "200":
+          description: Returns a CSV as a JSON String.
+          content:
+            text/csv:
+              schema:
+                type: string
+      tags:
+        - Visualization Operations

api/routes.py

@@ -12,11 +12,14 @@
 
 from schematic import CONFIG
 
+from schematic.visualization.attributes_explorer import AttributesExplorer
+from schematic.visualization.tangled_tree import TangledTree
 from schematic.manifest.generator import ManifestGenerator
 from schematic.models.metadata import MetadataModel
 from schematic.schemas.generator import SchemaGenerator
 from schematic.schemas.explorer import SchemaExplorer
 from schematic.store.synapse import SynapseStorage
+from flask_cors import CORS, cross_origin
 from schematic.schemas.explorer import SchemaExplorer
 import pandas as pd
 import json
@@ -349,7 +352,6 @@ def populate_manifest_route(schema_url, title=None, data_type=None):
 
     return populated_manifest_link
 
-
 def get_storage_projects(input_token, asset_view):
     # call config handler 
     config_handler(asset_view=asset_view)
@@ -374,7 +376,6 @@ def get_storage_projects_datasets(input_token, asset_view, project_id):
 
     return sorted_dataset_lst
 
-
 def get_files_storage_dataset(input_token, asset_view, dataset_id, full_path, file_names=None):
     # call config handler
     config_handler(asset_view=asset_view)
@@ -389,13 +390,48 @@ def get_files_storage_dataset(input_token, asset_view, dataset_id, full_path, fi
     # call getFilesInStorageDataset function
     file_lst = store.getFilesInStorageDataset(datasetId=dataset_id, fileNames=file_names, fullpath=full_path)
     return file_lst
+
 def get_component_requirements(schema_url, source_component, as_graph):
     metadata_model = initalize_metadata_model(schema_url)
 
     req_components = metadata_model.get_component_requirements(source_component=source_component, as_graph = as_graph)
 
     return req_components
 
+def get_viz_attributes_explorer(schema_url):
+    # call config_handler()
+    config_handler()
+
+    temp_path_to_jsonld = get_temp_jsonld(schema_url)
+
+    attributes_csv = AttributesExplorer(temp_path_to_jsonld).parse_attributes(save_file=False)
+
+    return attributes_csv
+
+def get_viz_tangled_tree_text(schema_url, figure_type, text_format):
+
+    temp_path_to_jsonld = get_temp_jsonld(schema_url)
+
+    # Initialize TangledTree
+    tangled_tree = TangledTree(temp_path_to_jsonld, figure_type)
+
+    # Get text for tangled tree.
+    text_df = tangled_tree.get_text_for_tangled_tree(text_format, save_file=False)
+
+    return text_df
+
+def get_viz_tangled_tree_layers(schema_url, figure_type):
+
+    temp_path_to_jsonld = get_temp_jsonld(schema_url)
+
+    # Initialize Tangled Tree
+    tangled_tree = TangledTree(temp_path_to_jsonld, figure_type)
+
+    # Get tangled trees layers JSON.
+    layers = tangled_tree.get_tangled_tree_layers(save_file=False)
+
+    return layers[0]
+
 def download_manifest(input_token, dataset_id, asset_view, as_json, new_manifest_name=''):
     # call config handler
     config_handler(asset_view=asset_view)

pyproject.toml

@@ -65,6 +65,7 @@ itsdangerous = "1.1.0"
 Jinja2 = "2.11.3"
 openpyxl = "^3.0.9"
 "backports.zoneinfo" = {markers = "python_version < \"3.9\"", version = "^0.2.1"}
+Flask-Cors = "^3.0.10"
 
 
 [tool.poetry.dev-dependencies]

run_api.py

@@ -3,6 +3,7 @@
 # import our application
 # Run our application
 from api import create_app
+from flask_cors import CORS
 import os
 
 if __name__ == "__main__":
@@ -13,4 +14,5 @@
 
     # Launch app
     app = create_app()
-    app.run(host=host, port=port, debug=True)
+    CORS(app, resources={r"*": {"origins": "*"}})
+    app.run(port=3001, debug=True)

schematic/__main__.py

@@ -11,6 +11,9 @@
 from schematic.schemas.commands import (
     schema as schema_cli,
 )  # schema conversion commands
+from schematic.visualization.commands import (
+    viz as viz_cli,
+)  # viz generation commands
 from schematic import init as init_cli  # schematic initialization commands
 
 logger = logging.getLogger()
@@ -34,6 +37,8 @@ def main():
 main.add_command(manifest_cli)  # add manifest commands
 main.add_command(model_cli)  # add model commands
 main.add_command(schema_cli)  # add schema commands
+main.add_command(viz_cli)  # add viz commands
+
 
 
 if __name__ == "__main__":

schematic/help.py

@@ -178,3 +178,18 @@
         ),
     }
 }
+
+viz_commands = {
+    "visualization": {
+        "config": (
+            "Specify the path to the `config.yml` using this option. This is a required argument."
+        ),
+        "tangled_tree": {
+            "figure_type": ("Specify the type of schema visualization to make. Either 'dependency' or 'component'."
+                ),
+            "text_format": ("Specify the type of text to gather for tangled tree visualization, either 'plain' or 'highlighted'."
+                ),
+        }
+    }
+
+}

schematic/schemas/explorer.py

@@ -617,6 +617,26 @@ def update_property(self, property_info):
         validate_schema(self.schema)
         logger.info(f"Updated the property {property_info['rdfs:label']} successfully.")
 
+    def get_nodes_descendants(self, graph, component):
+        """
+        Return a list of nodes reachable from source in graph 
+        graph: networkx graph object
+        component: any given node
+        """
+        all_descendants = list(nx.descendants(graph, component))
+
+        return all_descendants
+
+    def get_nodes_ancestors(self, graph, component):
+        """
+        Return a list of nodes reachable from source in graph 
+        graph: networkx graph object
+        component: any given node
+        """
+        all_ancestors = list(nx.ancestors(graph, component))
+
+        return all_ancestors
+
     def get_digraph_by_edge_type(self, edge_type):
 
         multi_digraph = self.schema_nx

schematic/schemas/generator.py

@@ -1,3 +1,4 @@
+import gc
 import os
 import json
 import logging

schematic/store/synapse.py

@@ -1437,7 +1437,7 @@ def make_synapse_table(self,
 
                         if col in table_schema_by_cname:
                             col_type = table_schema_by_cname[col]['columnType']
-                            max_size = table_schema_by_cname[col]['maximumSize']
+                            max_size = table_schema_by_cname[col]['maximumSize'] if 'maximumSize' in table_schema_by_cname[col].keys() else 100
                             max_list_len = 250
                             if max_size and max_list_len:
                                 cols.append(Column(name=col, columnType=col_type, 
@@ -1487,7 +1487,7 @@ def make_synapse_table(self,
                 for col in table_to_load.columns:
                     if col in table_schema_by_cname:
                         col_type = table_schema_by_cname[col]['columnType']
-                        max_size = table_schema_by_cname[col]['maximumSize']
+                        max_size = table_schema_by_cname[col]['maximumSize'] if 'maximumSize' in table_schema_by_cname[col].keys() else 100
                         max_list_len = 250
                         if max_size and max_list_len:
                             cols.append(Column(name=col, columnType=col_type, 

schematic/visualization/__init__.py

@@ -0,0 +1,2 @@
+from schematic.visualization.attributes_explorer import AttributesExplorer
+from schematic.visualization.tangled_tree import TangledTree