MRG: Merge pull request #81 from octue/fix/schemas

Fix: Schemas
octue · May 21, 2021 · 3eb5f12 · 3eb5f12
2 parents f2993c5 + 3b3155b
commit 3eb5f12
Show file tree

Hide file tree

Showing 17 changed files with 69 additions and 128 deletions.
diff --git a/docs/source/examples.rst b/docs/source/examples.rst
@@ -47,7 +47,7 @@ copied straight from the unit test cases, so you can always check there to see h
              "description": "This twine helps compute the cost of an installed foundation.",
              "children": [
              ],
-             "configuration_schema": {
+             "configuration_values_schema": {
                  "$schema": "http://json-schema.org/2019-09/schema#",
                  "title": "Foundation cost twin configuration",
                  "description": "Set config parameters and constants at startup of the twin.",
@@ -128,8 +128,7 @@ copied straight from the unit test cases, so you can always check there to see h
          	"input_manifest": [
          		{
          			"key": "wind_resource_data",
-         			"purpose": "A dataset containing Wind Resource Grid files",
-         			"filters": "tags:(wind AND resource) files:(extension:wrg)"
+         			"purpose": "A dataset containing Wind Resource Grid files"
          		}
          	],
          	"input_values_schema": {
@@ -184,4 +183,4 @@ copied straight from the unit test cases, so you can always check there to see h
          			}
          		}
          	}
-         }
+         }
diff --git a/examples/damage_classifier_service/data/configuration_manifest.json b/examples/damage_classifier_service/data/configuration_manifest.json
@@ -5,14 +5,14 @@
       "id": "7ead7669-8162-4f64-8cd5-4abe92509e17",
       "name": "training data for system abc123",
       "organisation": "megacorp",
-      "tags": "classifier, damage, system:abc123",
+      "tags": ["classifier", "damage", "system:abc123"],
       "files": [
         {
           "path": "datasets/7ead7669/blade_damage.mdl",
           "cluster": 0,
           "sequence": 0,
           "extension": "csv",
-          "tags": "",
+          "tags": [],
           "posix_timestamp": 0,
           "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86",
           "last_modified": "2019-02-28T22:40:30.533005Z",

diff --git a/examples/damage_classifier_service/twine.json b/examples/damage_classifier_service/twine.json
@@ -5,10 +5,7 @@
       // Once the inputs are validated, your analysis program can use this key to access the dataset
       "key": "trained_model",
       // General notes, which are helpful as a reminder to users of the service
-      "purpose": "The trained classifier",
-      // Issues a strict search for data provided by megacorp, containing *.mdl files tagged as
-      // classifiers for blade damage on system abc123
-      "filters": "organisation: megacorp AND tags:(classifier AND damage AND system:abc123) AND files:(extension:mdl)"
+      "purpose": "The trained classifier"
     }
   ]
 }
diff --git a/examples/met_mast_scada_service/data/input_manifest.json b/examples/met_mast_scada_service/data/input_manifest.json
@@ -4,14 +4,14 @@
     {
       "id": "7ead7669-8162-4f64-8cd5-4abe92509e17",
       "name": "meteorological mast dataset",
-      "tags": "met, mast, wind, location:108346",
+      "tags": ["met", "mast", "wind", "location:108346"],
       "files": [
         {
           "path": "input/datasets/7ead7669/mast_1.csv",
           "cluster": 0,
           "sequence": 0,
           "extension": "csv",
-          "tags": "",
+          "tags": [],
           "posix_timestamp": 1551393630,
           "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86",
           "last_modified": "2019-02-28T22:40:30.533005Z",
@@ -24,7 +24,7 @@
           "cluster": 0,
           "sequence": 1,
           "extension": "csv",
-          "tags": "",
+          "tags": [],
           "posix_timestamp": 1551394230,
           "id": "bbff07bc-7c19-4ed5-be6d-a6546eae8e45",
           "last_modified": "2019-02-28T22:50:40.633001Z",
@@ -37,14 +37,14 @@
     {
       "id": "5cf9e445-c288-4567-9072-edc31003b022",
       "name": "scada data exports",
-      "tags": "wind, turbine, scada, system:ab32, location:108346",
+      "tags": ["wind", "turbine", "scada", "system:ab32", "location:108346"],
       "files": [
         {
           "path": "input/datasets/7ead7669/export_1.csv",
           "cluster": 0,
           "sequence": 0,
           "extension": "csv",
-          "tags": "",
+          "tags": [],
           "posix_timestamp": 1551393600,
           "id": "78fa511f-3e28-4bc2-aa28-7b6a2e8e6ef9",
           "last_modified": "2019-02-28T22:40:00.000000Z",
@@ -57,7 +57,7 @@
           "cluster": 0,
           "sequence": 1,
           "extension": "csv",
-          "tags": "",
+          "tags": [],
           "posix_timestamp": 1551394200,
           "id": "204d7316-7ae6-45e3-8f90-443225b21226",
           "last_modified": "2019-02-28T22:50:00.000000Z",

diff --git a/examples/met_mast_scada_service/data/output_manifest.json b/examples/met_mast_scada_service/data/output_manifest.json
@@ -5,14 +5,14 @@
       "id": "4564deca-5654-42e8-aadf-70690b393a30",
       "name": "visual cross check data",
       "organisation": "megacorp",
-      "tags": "figure, met, mast, scada, check, location:108346",
+      "tags": ["figure", "met", "mast", "scada", "check", "location:108346"],
       "files": [
         {
           "path": "datasets/7ead7669/cross_check.fig",
           "cluster": 0,
           "sequence": 0,
           "extension": "fig",
-          "tags": "",
+          "tags": [],
           "posix_timestamp": 1551394800,
           "id": "38f77fe2-c8c0-49d1-a08c-0928d53a742f",
           "last_modified": "2019-02-28T23:00:00.000000Z",

diff --git a/examples/met_mast_scada_service/strands/input_manifest_filters.json b/examples/met_mast_scada_service/strands/input_manifest_filters.json
@@ -5,16 +5,11 @@
       // Once the inputs are validated, your analysis program can use this key to access the dataset
       "key": "met_mast_data",
       // General notes, which are helpful as a reminder to users of the service
-      "purpose": "A dataset containing meteorological mast data",
-      // Searches datasets which are tagged "met*" (allowing for "met" and "meterological"), whose
-      // files are CSVs in a numbered sequence, and which occur at a particular location
-      "filters": "tags:(met* AND mast) AND files:(extension:csv AND sequence:>=0) AND location:10"
+      "purpose": "A dataset containing meteorological mast data"
     },
     {
       "key": "scada_data",
-      "purpose": "A dataset containing scada data",
-      // The organisation: filter refines search to datasets owned by a particular organisation handle
-      "filters": "organisation: megacorp AND tags:(scada AND mast) AND files:(extension:csv AND sequence:>=0)"
+      "purpose": "A dataset containing scada data"
     }
-  ],
+  ]
 }
diff --git a/examples/met_mast_scada_service/strands/output_manifest_filters.json b/examples/met_mast_scada_service/strands/output_manifest_filters.json
@@ -4,9 +4,7 @@
       // Twined will prepare a manifest with this key, which you can add to during the analysis or once its complete
       "key": "met_scada_checks",
       // General notes, which are helpful as a reminder to users of the service
-      "purpose": "A dataset containing figures showing correlations between mast and scada data",
-      // Twined will check that the output file manifest has tags appropriate to the filters
-      "filters": "tags:(met* AND scada AND correlation) AND files:(extension:json) AND location:*"
+      "purpose": "A dataset containing figures showing correlations between mast and scada data"
     }
   ]
 }
diff --git a/examples/wind_tunnel_datalogger_service/data/output_manifest.json b/examples/wind_tunnel_datalogger_service/data/output_manifest.json
@@ -5,14 +5,14 @@
       "id": "1eba4346-daff-421b-921c-8f1c05d6997d",
       "name": "Test results from naca0012 section",
       "organisation": "megacorp",
-      "tags": "section:naca0012",
+      "tags": ["section:naca0012"],
       "files": [
         {
           "path": "datasets/7ead7669/sys_temp.json",
           "cluster": 0,
           "sequence": 0,
           "extension": "json",
-          "tags": "system, temperature",
+          "tags": ["system", "temperature"],
           "posix_timestamp": 1551394800,
           "id": "afcdef45-da6b-4805-95d6-7a889d81f5b9",
           "last_modified": "2020-02-28T13:12:42.000000Z",
@@ -25,7 +25,7 @@
           "cluster": 1,
           "sequence": 0,
           "extension": "json",
-          "tags": "wind, tunnel, velocity, profile, background, turbulence",
+          "tags": ["wind", "tunnel", "velocity", "profile", "background", "turbulence"],
           "posix_timestamp": 1551394800,
           "id": "3667aa6d-ee64-4cd4-a2fd-e72bcdc65791",
           "last_modified": "2020-02-28T13:24:43.000000Z",
@@ -38,7 +38,7 @@
           "cluster": 2,
           "sequence": 0,
           "extension": "dat",
-          "tags": "pressure, coefficient, cp, profile, reference",
+          "tags": ["pressure", "coefficient", "cp", "profile", "reference"],
           "posix_timestamp": 1551394800,
           "id": "310bc665-fe8c-4948-b821-0ad43fcd480d",
           "last_modified": "2020-02-28T13:47:23.000000Z",
@@ -51,7 +51,7 @@
           "cluster": 3,
           "sequence": 0,
           "extension": "dat",
-          "tags": "pressure, coefficient, cp, profile, reference, alpha:0",
+          "tags": ["pressure", "coefficient", "cp", "profile", "reference", "alpha:0"],
           "posix_timestamp": 1551394800,
           "id": "c3a6c14d-19d8-44da-9aa5-119798f53d15",
           "last_modified": "2020-02-28T13:54:24.000000Z",
@@ -64,7 +64,7 @@
           "cluster": 3,
           "sequence": 1,
           "extension": "dat",
-          "tags": "pressure, coefficient, cp, profile, reference, alpha:1",
+          "tags": ["pressure", "coefficient", "cp", "profile", "reference", "alpha:1"],
           "posix_timestamp": 1551394800,
           "id": "fac62036-722c-481a-9daf-87897c4872ec",
           "last_modified": "2020-02-28T13:56:21.000000Z",
@@ -77,7 +77,7 @@
           "cluster": 3,
           "sequence": 2,
           "extension": "dat",
-          "tags": "pressure, coefficient, cp, profile, reference, alpha:2",
+          "tags": ["pressure", "coefficient", "cp", "profile", "reference", "alpha:2"],
           "posix_timestamp": 1551394800,
           "id": "70cda7f6-c97d-4b99-9156-2ff6f5947b7e",
           "last_modified": "2020-02-28T13:57:03.000000Z",
@@ -90,7 +90,7 @@
           "cluster": 3,
           "sequence": 3,
           "extension": "dat",
-          "tags": "pressure, coefficient, cp, profile, reference, alpha:3",
+          "tags": ["pressure", "coefficient", "cp", "profile", "reference", "alpha:3"],
           "posix_timestamp": 1551394800,
           "id": "5ab4015a-608a-4ecd-9e30-95aee82d86d9",
           "last_modified": "2020-02-28T13:58:46.000000Z",
@@ -103,7 +103,7 @@
           "cluster": 3,
           "sequence": 4,
           "extension": "dat",
-          "tags": "pressure, coefficient, cp, profile, reference, alpha:4",
+          "tags": ["pressure", "coefficient", "cp", "profile", "reference", "alpha:4"],
           "posix_timestamp": 1551394800,
           "id": "3ba97d4b-002d-4ca3-a6b0-54573a5eefde",
           "last_modified": "2020-02-28T13:59:32.000000Z",

diff --git a/examples/wind_tunnel_datalogger_service/strands/output_manifest_filters.json b/examples/wind_tunnel_datalogger_service/strands/output_manifest_filters.json
@@ -4,9 +4,7 @@
       // Twined will prepare a manifest with this key, which you can add to during the analysis or once its complete
       "key": "met_scada_checks",
       // General notes, which are helpful as a reminder to users of the service
-      "purpose": "A dataset containing figures showing correlations between mast and scada data",
-      // Twined will check that the output file manifest has tags appropriate to the filters
-      "filters": "tags:(met* AND scada AND correlation) AND files:(extension:json) AND location:*"
+      "purpose": "A dataset containing figures showing correlations between mast and scada data"
     }
   ]
 }
diff --git a/tests/data/apps/empty_app/twine.json b/tests/data/apps/empty_app/twine.json
@@ -1,7 +1,7 @@
 {
 	"children": [
 	],
-	"configuration_schema": {
+	"configuration_values_schema": {
 		"$schema": "http://json-schema.org/2019-09/schema#",
 		"title": "The example configuration form",
 		"description": "The configuration strand of an example twine",

diff --git a/tests/data/apps/example_app/input/manifest.json b/tests/data/apps/example_app/input/manifest.json
@@ -5,14 +5,14 @@
     {
       "id": "7ead7669-8162-4f64-8cd5-4abe92509e17",
       "name": "my meteorological dataset",
-      "tags": "met, mast, wind",
+      "tags": ["met", "mast", "wind"],
       "files": [
         {
           "path": "input/datasets/7ead7669/file_1.csv",
           "cluster": 0,
           "sequence": 0,
           "extension": "csv",
-          "tags": "",
+          "tags": [],
           "posix_timestamp": null,
           "data_file": {
             "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86",
@@ -27,7 +27,7 @@
           "cluster": 0,
           "sequence": 1,
           "extension": "csv",
-          "tags": "",
+          "tags": [],
           "posix_timestamp": null,
           "data_file": {
             "id": "bbff07bc-7c19-4ed5-be6d-a6546eae8e45",

diff --git a/tests/data/apps/example_app/twine.json b/tests/data/apps/example_app/twine.json
@@ -6,7 +6,7 @@
 			"filters": "tags:(met* AND mast AND location) files:(extension:csv AND sequence:>=0) location:10"
 		}
 	],
-	"configuration_schema": {
+	"configuration_values_schema": {
 		"$schema": "http://json-schema.org/2019-09/schema#",
 		"title": "The example configuration form",
 		"description": "The configuration strand of an example twine",
@@ -34,13 +34,11 @@
 	"input_manifest": [
 		{
 			"key": "met_mast_data",
-			"purpose": "A dataset containing meteorological mast data",
-			"filters": "tags:(met* AND mast AND location) files:(extension:csv AND sequence:>=0) location:10"
+			"purpose": "A dataset containing meteorological mast data"
 		},
 		{
 			"key": "scada_data",
-			"purpose": "A dataset containing scada data",
-			"filters": "tags:(met* AND mast) files:(extension:csv AND sequence:>=0) location:10"
+			"purpose": "A dataset containing scada data"
 		}
 	],
 	"input_values_schema": {

diff --git a/tests/data/apps/simple_app/twine.json b/tests/data/apps/simple_app/twine.json
@@ -1,5 +1,5 @@
 {
-	"configuration_schema": {
+	"configuration_values_schema": {
 		"$schema": "http://json-schema.org/2019-09/schema#",
 		"title": "Configuration for a simple app",
 		"description": "The app creates a mandelbrot plot",

diff --git a/tests/test_manifest_strands.py b/tests/test_manifest_strands.py
@@ -13,27 +13,23 @@ class TestManifestStrands(BaseTestCase):
             "configuration_manifest": [
                 {
                     "key": "configuration_files_data",
-                    "purpose": "A dataset containing files used in configuration",
-                    "filters": "tags:(the AND config AND tags) files:(extension:csv AND sequence:>=0)"
+                    "purpose": "A dataset containing files used in configuration"
                 }
             ],
             "input_manifest": [
                 {
                     "key": "met_mast_data",
-                    "purpose": "A dataset containing meteorological mast data",
-                    "filters": "tags:(met* AND mast AND location) files:(extension:csv AND sequence:>=0) location:10"
+                    "purpose": "A dataset containing meteorological mast data"
                 },
                 {
                     "key": "scada_data",
-                    "purpose": "A dataset containing scada data",
-                    "filters": "tags:(met* AND mast) files:(extension:csv AND sequence:>=0) location:10"
+                    "purpose": "A dataset containing scada data"
                 }
             ],
             "output_manifest": [
                 {
                     "key": "output_files_data",
-                    "purpose": "A dataset containing output results",
-                    "filters": "tags:(the AND output AND tags) files:(extension:csv AND sequence:>=0)"
+                    "purpose": "A dataset containing output results"
                 }
             ]
         }
@@ -179,7 +175,7 @@ def test_valid_manifest_files(self):
         """
 
         twine = Twine(source=self.VALID_MANIFEST_STRAND)
-        twine.validate_input_manifest(source=valid_configuration_manifest)
+        twine.validate_configuration_manifest(source=valid_configuration_manifest)
         twine.validate_input_manifest(source=valid_input_manifest)
         twine.validate_output_manifest(source=valid_output_manifest)
 

diff --git a/tests/test_twine.py b/tests/test_twine.py
@@ -49,7 +49,7 @@ def test_broken_json_twine(self):
         invalid_json_twine = """
             {
                 "children": [
-                "configuration_schema": {
+                "configuration_values_schema": {
                     "$schema": "http://json-schema.org/2019-09/schema#",
                     "title": "The example configuration form",
                     "description": "The configuration strand of an example twine",

diff --git a/twined/schema/manifest_schema.json b/twined/schema/manifest_schema.json
@@ -1,20 +1,6 @@
 {
   "type": "object",
   "properties": {
-    "kind": {
-      "description": "The kind of the manifest, (only 'multi-dataset' available, reserved for future use)",
-      "type": "string",
-      "default": "multi-dataset",
-      "anyOf": [
-        {
-          "type": "string",
-          "enum": [
-            "multi-dataset"
-          ],
-          "title": "Multi Dataset"
-        }
-      ]
-    },
     "id": {
       "description": "ID of the manifest, typically a uuid",
       "type": "string"