MRG: Merge pull request #83 from octue/feature/tag-templates-2

Feature: Tag templates
octue · Jun 2, 2021 · 6d80a61 · 6d80a61
2 parents 3eb5f12 + 9fee567
commit 6d80a61
Show file tree

Hide file tree

Showing 23 changed files with 941 additions and 275 deletions.
diff --git a/docs/source/anatomy_credentials.rst b/docs/source/anatomy_credentials.rst
@@ -20,7 +20,7 @@ Defining the Credentials Strand
 ===============================
 
 This is the simplest of the strands, containing a list of credentials (whose ``NAMES_SHOULD_BE_SHOUTY_SNAKE_CASE``) with
-a reminder of the purpose. Defaults can also be provided, useful for running on local or closed networks.
+a reminder of the purpose.
 
 .. code-block:: javascript
 
@@ -36,8 +36,7 @@ a reminder of the purpose. Defaults can also be provided, useful for running on
        },
        {
          "name": "SECRET_THE_THIRD",
-         "purpose": "Usually a big secret but sometimes has a convenient non-secret default, like a sandbox or local database",
-         "default": "postgres://pguser:pgpassword@localhost:5432/pgdb"
+         "purpose": "Another secret, like a password for a sandbox or local database"
        }
      ]
    }

diff --git a/docs/source/anatomy_manifest.rst b/docs/source/anatomy_manifest.rst
diff --git a/docs/source/examples.rst b/docs/source/examples.rst
@@ -81,8 +81,9 @@ copied straight from the unit test cases, so you can always check there to see h
                      }
                  }
              },
-             "output_manifest": [
-             ],
+             "output_manifest": {
+                "datasets": []
+             },
              "output_values_schema": {
                  "title": "Output Values schema for the foundation cost twin",
                  "description": "The response supplied to a change in input values will always conform to this schema.",
@@ -125,12 +126,14 @@ copied straight from the unit test cases, so you can always check there to see h
          			"purpose": "Token for accessing a 3rd party weather API service"
          		}
          	],
-         	"input_manifest": [
-         		{
-         			"key": "wind_resource_data",
-         			"purpose": "A dataset containing Wind Resource Grid files"
-         		}
-         	],
+         	"input_manifest": {
+                "datasets": [
+                    {
+                        "key": "wind_resource_data",
+                        "purpose": "A dataset containing Wind Resource Grid files"
+                    }
+                ]
+            },
          	"input_values_schema": {
          		"$schema": "http://json-schema.org/2019-09/schema#",
          		"title": "Input Values for the weather service twin",
@@ -160,13 +163,16 @@ copied straight from the unit test cases, so you can always check there to see h
          			}
          		}
          	},
-         	"output_manifest": [
-         		{
-         			"key": "production_data",
-         			"purpose": "A dataset containing production data",
-         			"tags": "production, wind"
-         		}
-         	],
+         	"output_manifest": {
+                "datasets": [
+                    {
+                        "key": "production_data",
+                        "purpose": "A dataset containing production data",
+                        "tags": {"cleaned": true},
+                        "labels": ["production", "wind"]
+                    }
+                ]
+            },
          	"output_values_schema": {
          		"$schema": "http://json-schema.org/2019-09/schema#",
          		"title": "Output Values for the metocean service twin",

diff --git a/examples/damage_classifier_service/data/configuration_manifest.json b/examples/damage_classifier_service/data/configuration_manifest.json
@@ -5,14 +5,16 @@
       "id": "7ead7669-8162-4f64-8cd5-4abe92509e17",
       "name": "training data for system abc123",
       "organisation": "megacorp",
-      "tags": ["classifier", "damage", "system:abc123"],
+      "tags": {"system": "abc123"},
+      "labels": ["classifier", "damage"],
       "files": [
         {
           "path": "datasets/7ead7669/blade_damage.mdl",
           "cluster": 0,
           "sequence": 0,
           "extension": "csv",
-          "tags": [],
+          "tags": {},
+          "labels": [],
           "posix_timestamp": 0,
           "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86",
           "last_modified": "2019-02-28T22:40:30.533005Z",

diff --git a/examples/damage_classifier_service/twine.json b/examples/damage_classifier_service/twine.json
@@ -1,11 +1,13 @@
 {
   // Manifest strands contain lists, with one entry for each required dataset
-  "configuration_manifest": [
-    {
-      // Once the inputs are validated, your analysis program can use this key to access the dataset
-      "key": "trained_model",
-      // General notes, which are helpful as a reminder to users of the service
-      "purpose": "The trained classifier"
-    }
-  ]
+  "configuration_manifest": {
+    "datasets": [
+      {
+        // Once the inputs are validated, your analysis program can use this key to access the dataset
+        "key": "trained_model",
+        // General notes, which are helpful as a reminder to users of the service
+        "purpose": "The trained classifier"
+      }
+    ]
+  }
 }
diff --git a/examples/met_mast_scada_service/data/input_manifest.json b/examples/met_mast_scada_service/data/input_manifest.json
@@ -4,14 +4,16 @@
     {
       "id": "7ead7669-8162-4f64-8cd5-4abe92509e17",
       "name": "meteorological mast dataset",
-      "tags": ["met", "mast", "wind", "location:108346"],
+      "tags": {"location": 108346},
+      "labels": ["met", "mast", "wind"],
       "files": [
         {
           "path": "input/datasets/7ead7669/mast_1.csv",
           "cluster": 0,
           "sequence": 0,
           "extension": "csv",
-          "tags": [],
+          "tags": {},
+          "labels": [],
           "posix_timestamp": 1551393630,
           "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86",
           "last_modified": "2019-02-28T22:40:30.533005Z",
@@ -24,7 +26,8 @@
           "cluster": 0,
           "sequence": 1,
           "extension": "csv",
-          "tags": [],
+          "tags": {},
+          "labels": [],
           "posix_timestamp": 1551394230,
           "id": "bbff07bc-7c19-4ed5-be6d-a6546eae8e45",
           "last_modified": "2019-02-28T22:50:40.633001Z",
@@ -37,14 +40,16 @@
     {
       "id": "5cf9e445-c288-4567-9072-edc31003b022",
       "name": "scada data exports",
-      "tags": ["wind", "turbine", "scada", "system:ab32", "location:108346"],
+      "tags": {"location": 108346, "system": "ab32"},
+      "labels": ["wind", "turbine", "scada"],
       "files": [
         {
           "path": "input/datasets/7ead7669/export_1.csv",
           "cluster": 0,
           "sequence": 0,
           "extension": "csv",
-          "tags": [],
+          "tags": {},
+          "labels": [],
           "posix_timestamp": 1551393600,
           "id": "78fa511f-3e28-4bc2-aa28-7b6a2e8e6ef9",
           "last_modified": "2019-02-28T22:40:00.000000Z",
@@ -57,7 +62,8 @@
           "cluster": 0,
           "sequence": 1,
           "extension": "csv",
-          "tags": [],
+          "tags": {},
+          "labels": [],
           "posix_timestamp": 1551394200,
           "id": "204d7316-7ae6-45e3-8f90-443225b21226",
           "last_modified": "2019-02-28T22:50:00.000000Z",

diff --git a/examples/met_mast_scada_service/data/output_manifest.json b/examples/met_mast_scada_service/data/output_manifest.json
@@ -5,14 +5,16 @@
       "id": "4564deca-5654-42e8-aadf-70690b393a30",
       "name": "visual cross check data",
       "organisation": "megacorp",
-      "tags": ["figure", "met", "mast", "scada", "check", "location:108346"],
+      "tags": {"location": 108346},
+      "labels": ["figure", "met", "mast", "scada", "check"],
       "files": [
         {
           "path": "datasets/7ead7669/cross_check.fig",
           "cluster": 0,
           "sequence": 0,
           "extension": "fig",
-          "tags": [],
+          "tags": {},
+          "labels": [],
           "posix_timestamp": 1551394800,
           "id": "38f77fe2-c8c0-49d1-a08c-0928d53a742f",
           "last_modified": "2019-02-28T23:00:00.000000Z",

diff --git a/examples/met_mast_scada_service/strands/input_manifest.json b/examples/met_mast_scada_service/strands/input_manifest.json
@@ -0,0 +1,17 @@
+{
+  // Manifest strands contain lists, with one entry for each required dataset
+  "input_manifest": {
+    "datasets": [
+      {
+        // Once the inputs are validated, your analysis program can use this key to access the dataset
+        "key": "met_mast_data",
+        // General notes, which are helpful as a reminder to users of the service
+        "purpose": "A dataset containing meteorological mast data"
+      },
+      {
+        "key": "scada_data",
+        "purpose": "A dataset containing scada data"
+      }
+    ]
+  }
+}
diff --git a/examples/met_mast_scada_service/strands/input_manifest_filters.json b/examples/met_mast_scada_service/strands/input_manifest_filters.json
diff --git a/examples/met_mast_scada_service/strands/output_manifest.json b/examples/met_mast_scada_service/strands/output_manifest.json
@@ -0,0 +1,12 @@
+{
+  "output_manifest": {
+    "datasets": [
+      {
+        // Twined will prepare a manifest with this key, which you can add to during the analysis or once its complete
+        "key": "met_scada_checks",
+        // General notes, which are helpful as a reminder to users of the service
+        "purpose": "A dataset containing figures showing correlations between mast and scada data"
+      }
+    ]
+  }
+}
diff --git a/examples/met_mast_scada_service/strands/output_manifest_filters.json b/examples/met_mast_scada_service/strands/output_manifest_filters.json
diff --git a/examples/wind_tunnel_datalogger_service/data/output_manifest.json b/examples/wind_tunnel_datalogger_service/data/output_manifest.json
@@ -5,14 +5,16 @@
       "id": "1eba4346-daff-421b-921c-8f1c05d6997d",
       "name": "Test results from naca0012 section",
       "organisation": "megacorp",
-      "tags": ["section:naca0012"],
+      "tags": {"section": "naca0012"},
+      "labels": [],
       "files": [
         {
           "path": "datasets/7ead7669/sys_temp.json",
           "cluster": 0,
           "sequence": 0,
           "extension": "json",
-          "tags": ["system", "temperature"],
+          "tags": {},
+          "labels": ["system", "temperature"],
           "posix_timestamp": 1551394800,
           "id": "afcdef45-da6b-4805-95d6-7a889d81f5b9",
           "last_modified": "2020-02-28T13:12:42.000000Z",
@@ -25,7 +27,8 @@
           "cluster": 1,
           "sequence": 0,
           "extension": "json",
-          "tags": ["wind", "tunnel", "velocity", "profile", "background", "turbulence"],
+          "tags": {},
+          "labels": ["wind", "tunnel", "velocity", "profile", "background", "turbulence"],
           "posix_timestamp": 1551394800,
           "id": "3667aa6d-ee64-4cd4-a2fd-e72bcdc65791",
           "last_modified": "2020-02-28T13:24:43.000000Z",
@@ -38,7 +41,8 @@
           "cluster": 2,
           "sequence": 0,
           "extension": "dat",
-          "tags": ["pressure", "coefficient", "cp", "profile", "reference"],
+          "tags": {},
+          "labels": ["pressure", "coefficient", "cp", "profile", "reference"],
           "posix_timestamp": 1551394800,
           "id": "310bc665-fe8c-4948-b821-0ad43fcd480d",
           "last_modified": "2020-02-28T13:47:23.000000Z",
@@ -51,7 +55,8 @@
           "cluster": 3,
           "sequence": 0,
           "extension": "dat",
-          "tags": ["pressure", "coefficient", "cp", "profile", "reference", "alpha:0"],
+          "tags": {"alpha": 0},
+          "labels": ["pressure", "coefficient", "cp", "profile", "reference"],
           "posix_timestamp": 1551394800,
           "id": "c3a6c14d-19d8-44da-9aa5-119798f53d15",
           "last_modified": "2020-02-28T13:54:24.000000Z",
@@ -64,7 +69,8 @@
           "cluster": 3,
           "sequence": 1,
           "extension": "dat",
-          "tags": ["pressure", "coefficient", "cp", "profile", "reference", "alpha:1"],
+          "tags": {"alpha": 1},
+          "labels": ["pressure", "coefficient", "cp", "profile", "reference"],
           "posix_timestamp": 1551394800,
           "id": "fac62036-722c-481a-9daf-87897c4872ec",
           "last_modified": "2020-02-28T13:56:21.000000Z",
@@ -77,7 +83,8 @@
           "cluster": 3,
           "sequence": 2,
           "extension": "dat",
-          "tags": ["pressure", "coefficient", "cp", "profile", "reference", "alpha:2"],
+          "tags": {"alpha": 2},
+          "labels": ["pressure", "coefficient", "cp", "profile", "reference"],
           "posix_timestamp": 1551394800,
           "id": "70cda7f6-c97d-4b99-9156-2ff6f5947b7e",
           "last_modified": "2020-02-28T13:57:03.000000Z",
@@ -90,7 +97,8 @@
           "cluster": 3,
           "sequence": 3,
           "extension": "dat",
-          "tags": ["pressure", "coefficient", "cp", "profile", "reference", "alpha:3"],
+          "tags": {"alpha": 3},
+          "labels": ["pressure", "coefficient", "cp", "profile", "reference"],
           "posix_timestamp": 1551394800,
           "id": "5ab4015a-608a-4ecd-9e30-95aee82d86d9",
           "last_modified": "2020-02-28T13:58:46.000000Z",
@@ -103,7 +111,8 @@
           "cluster": 3,
           "sequence": 4,
           "extension": "dat",
-          "tags": ["pressure", "coefficient", "cp", "profile", "reference", "alpha:4"],
+          "tags": {"alpha": 4},
+          "labels": ["pressure", "coefficient", "cp", "profile", "reference"],
           "posix_timestamp": 1551394800,
           "id": "3ba97d4b-002d-4ca3-a6b0-54573a5eefde",
           "last_modified": "2020-02-28T13:59:32.000000Z",

diff --git a/examples/wind_tunnel_datalogger_service/strands/output_manifest.json b/examples/wind_tunnel_datalogger_service/strands/output_manifest.json
@@ -0,0 +1,12 @@
+{
+  "output_manifest": {
+    "datasets": [
+      {
+        // Twined will prepare a manifest with this key, which you can add to during the analysis or once its complete
+        "key": "met_scada_checks",
+        // General notes, which are helpful as a reminder to users of the service
+        "purpose": "A dataset containing figures showing correlations between mast and scada data"
+      }
+    ]
+  }
+}
diff --git a/examples/wind_tunnel_datalogger_service/strands/output_manifest_filters.json b/examples/wind_tunnel_datalogger_service/strands/output_manifest_filters.json
diff --git a/tests/data/apps/empty_app/twine.json b/tests/data/apps/empty_app/twine.json
@@ -11,8 +11,9 @@
 	},
 	"credentials": [
 	],
-	"input_manifest": [
-	],
+	"input_manifest": {
+		"datasets": []
+	},
 	"input_values_schema": {
 		"$schema": "http://json-schema.org/2019-09/schema#",
 		"title": "Input Values",
@@ -21,8 +22,9 @@
 		"properties": {
 		}
 	},
-	"output_manifest": [
-	],
+	"output_manifest": {
+		"datasets": []
+	},
 	"output_values_schema": {
 		"title": "Output Values",
 		"description": "The output values strand of an example twine",