Merge pull request #7 from hydroshare/form-dict

add dictionary form compatibility in json schema

hsmodels/schemas/aggregations.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 from typing import List, Union
 
 from pydantic import AnyUrl, Field, root_validator, validator
@@ -26,6 +27,7 @@
     parse_multidimensional_spatial_reference,
     parse_spatial_coverage,
     parse_spatial_reference,
+    normalize_additional_metadata,
 )
 
 
@@ -74,6 +76,7 @@ class BaseAggregationMetadata(BaseMetadata):
 
     _language_constraint = validator('language', allow_reuse=True)(language_constraint)
     _parse_spatial_coverage = validator("spatial_coverage", allow_reuse=True, pre=True)(parse_spatial_coverage)
+    _normalize_additional_metadata = validator("additional_metadata", allow_reuse=True, pre=True)(normalize_additional_metadata)
 
 
 class GeographicRasterMetadata(BaseAggregationMetadata):
@@ -88,7 +91,8 @@ class GeographicRasterMetadata(BaseAggregationMetadata):
     class Config:
         title = 'Geographic Raster Aggregation Metadata'
 
-        schema_config = {'read_only': ['type', 'url']}
+        schema_config = {'read_only': ['type', 'url'],
+                         'dictionary_field': ['additional_metadata']}
 
     type: AggregationType = Field(
         const=True,
@@ -126,7 +130,8 @@ class GeographicFeatureMetadata(BaseAggregationMetadata):
     class Config:
         title = 'Geographic Feature Aggregation Metadata'
 
-        schema_config = {'read_only': ['type', 'url']}
+        schema_config = {'read_only': ['type', 'url'],
+                         'dictionary_field': ['additional_metadata']}
 
     type: AggregationType = Field(
         const=True,
@@ -165,7 +170,8 @@ class MultidimensionalMetadata(BaseAggregationMetadata):
     class Config:
         title = 'Multidimensional Aggregation Metadata'
 
-        schema_config = {'read_only': ['type', 'url']}
+        schema_config = {'read_only': ['type', 'url'],
+                         'dictionary_field': ['additional_metadata']}
 
     type: AggregationType = Field(
         const=True,
@@ -202,7 +208,8 @@ class ReferencedTimeSeriesMetadata(BaseAggregationMetadata):
     class Config:
         title = 'Referenced Time Series Aggregation Metadata'
 
-        schema_config = {'read_only': ['type', 'url']}
+        schema_config = {'read_only': ['type', 'url'],
+                         'dictionary_field': ['additional_metadata']}
 
     type: AggregationType = Field(
         const=True,
@@ -225,7 +232,8 @@ class FileSetMetadata(BaseAggregationMetadata):
     class Config:
         title = 'File Set Aggregation Metadata'
 
-        schema_config = {'read_only': ['type', 'url']}
+        schema_config = {'read_only': ['type', 'url'],
+                         'dictionary_field': ['additional_metadata']}
 
     type: AggregationType = Field(
         const=True,
@@ -247,7 +255,8 @@ class SingleFileMetadata(BaseAggregationMetadata):
     class Config:
         title = 'Single File Aggregation Metadata'
 
-        schema_config = {'read_only': ['type', 'url']}
+        schema_config = {'read_only': ['type', 'url'],
+                         'dictionary_field': ['additional_metadata']}
 
     type: AggregationType = Field(
         const=True,
@@ -272,7 +281,8 @@ class TimeSeriesMetadata(BaseAggregationMetadata):
     class Config:
         title = 'Time Series Aggregation Metadata'
 
-        schema_config = {'read_only': ['type', 'url']}
+        schema_config = {'read_only': ['type', 'url'],
+                         'dictionary_field': ['additional_metadata']}
 
     type: AggregationType = Field(
         const=True,

hsmodels/schemas/base_models.py

@@ -1,10 +1,39 @@
 from datetime import datetime
-from typing import Any, Dict, Type
+from typing import Any, Dict, Union
 
 from pydantic import BaseModel
 
 
 class BaseMetadata(BaseModel):
+
+    def dict(
+            self,
+            *,
+            include: Union['AbstractSetIntStr', 'MappingIntStrAny'] = None,
+            exclude: Union['AbstractSetIntStr', 'MappingIntStrAny'] = None,
+            by_alias: bool = False,
+            skip_defaults: bool = None,
+            exclude_unset: bool = False,
+            exclude_defaults: bool = False,
+            exclude_none: bool = False,
+    ) -> 'DictStrAny':
+        """
+        Checks the config for a schema_config dictionary_field and converts a dictionary to a list of key/value pairs.
+        This converts the dictionary to a format that can be described in a json schema (which can be found below in the
+        schema_extra staticmethod.
+
+        """
+        d = super().dict(include=include, exclude=exclude, by_alias=by_alias, skip_defaults=skip_defaults,
+                         exclude_unset=exclude_unset, exclude_defaults=exclude_defaults, exclude_none=exclude_none)
+
+        if hasattr(self.Config, "schema_config"):
+            schema_config = self.Config.schema_config
+            if "dictionary_field" in schema_config:
+                for field in schema_config["dictionary_field"]:
+                    field_value = d[field]
+                    d[field] = [{"key": key, "value": value} for key, value in field_value.items()]
+        return d
+
     class Config:
         validate_assignment = True
 
@@ -16,6 +45,30 @@ def schema_extra(schema: Dict[str, Any], model) -> None:
                     # set readOnly in json schema
                     for field in schema_config["read_only"]:
                         schema['properties'][field]['readOnly'] = True
+                if "dictionary_field" in schema_config:
+                    for field in schema_config["dictionary_field"]:
+                        prop = schema["properties"][field]
+                        prop.pop('default', None)
+                        prop.pop('additionalProperties', None)
+                        prop['type'] = "array"
+                        prop['items'] = \
+                            {
+                                "type": "object",
+                                "title": "Key-Value",
+                                "description": "A key-value pair",
+                                "properties": {
+                                    "key": {
+                                        "type": "string"
+                                    },
+                                    "value": {
+                                        "type": "string"
+                                    }
+                                },
+                                "required": [
+                                    "key",
+                                    "value"
+                                ]
+                            }
 
 
 class BaseCoverage(BaseMetadata):

hsmodels/schemas/rdf/root_validators.py

@@ -73,12 +73,8 @@ def rdf_parse_rdf_subject(cls, values):
 
 def parse_rdf_extended_metadata(cls, values):
     if "additional_metadata" in values:
-        em = values["additional_metadata"]
-        assert isinstance(em, dict)
-        values["extended_metadata"] = []
+        values["extended_metadata"] = values["additional_metadata"]
         del values["additional_metadata"]
-        for key, value in em.items():
-            values["extended_metadata"].append({"key": key, "value": value})
     return values
 
 

hsmodels/schemas/resource.py

@@ -23,7 +23,8 @@
     split_coverages,
     split_dates,
 )
-from hsmodels.schemas.validators import list_not_empty, parse_identifier, parse_sources, parse_spatial_coverage
+from hsmodels.schemas.validators import list_not_empty, parse_identifier, parse_sources, parse_spatial_coverage,\
+    normalize_additional_metadata
 
 
 class ResourceMetadata(BaseMetadata):
@@ -34,7 +35,8 @@ class ResourceMetadata(BaseMetadata):
     class Config:
         title = 'Resource Metadata'
 
-        schema_config = {'read_only': ['type', 'identifier', 'created', 'modified', 'published', 'url']}
+        schema_config = {'read_only': ['type', 'identifier', 'created', 'modified', 'published', 'url'],
+                         'dictionary_field': ['additional_metadata']}
 
     type: str = Field(
         const=True,
@@ -141,6 +143,7 @@ class Config:
     _parse_identifier = validator("identifier", pre=True)(parse_identifier)
     _parse_sources = validator("sources", pre=True)(parse_sources)
     _parse_spatial_coverage = validator("spatial_coverage", allow_reuse=True, pre=True)(parse_spatial_coverage)
+    _normalize_additional_metadata = validator("additional_metadata", allow_reuse=True, pre=True)(normalize_additional_metadata)
 
     _language_constraint = validator('language', allow_reuse=True)(language_constraint)
     _creators_constraint = validator('creators')(list_not_empty)

hsmodels/schemas/validators.py

@@ -52,6 +52,23 @@ def parse_spatial_coverage(cls, value):
     return value
 
 
+def normalize_additional_metadata(cls, value):
+    if isinstance(value, list):
+        as_dict = {}
+        for val in value:
+            if not isinstance(val, dict):
+                raise ValueError(f"List entry {val} must be a dict")
+            if "key" not in val:
+                raise ValueError(f"Missing the 'key' key in {val}")
+            if "value" not in val:
+                raise ValueError(f"Missing the 'value' key in {val}")
+            if val["key"] in as_dict:
+                raise ValueError(f"Found a duplicate key {val['key']}")
+            as_dict[val["key"]] = val["value"]
+        return as_dict
+    return value
+
+
 def list_not_empty(cls, l):
     if len(l) == 0:
         raise ValueError("list must contain at least one entry")

tests/test_schemas.py

@@ -58,3 +58,43 @@ def test_readonly(read_only_field):
             assert "readOnly" in s["properties"][prop] and s["properties"][prop]["readOnly"] is True
         else:
             assert "readOnly" not in s["properties"][prop]
+
+
+additional_metadata_fields = [
+    (ResourceMetadata, ['additional_metadata']),
+    (GeographicRasterMetadata, ['additional_metadata']),
+    (GeographicFeatureMetadata, ['additional_metadata']),
+    (MultidimensionalMetadata, ['additional_metadata']),
+    (ReferencedTimeSeriesMetadata, ['additional_metadata']),
+    (FileSetMetadata, ['additional_metadata']),
+    (SingleFileMetadata, ['additional_metadata']),
+    (TimeSeriesMetadata, ['additional_metadata']),
+]
+
+
+@pytest.mark.parametrize("additional_metadata_field", additional_metadata_fields)
+def test_dictionary_field(additional_metadata_field):
+    clazz, fields = additional_metadata_field
+    s = schema([clazz])["definitions"][clazz.__name__]
+
+    for field in fields:
+        assert 'additionalProperties' not in s["properties"][field]
+        assert 'default' not in s["properties"][field]
+        assert s["properties"][field]['items'] == \
+               {
+                   "type": "object",
+                   "title": "Key-Value",
+                   "description": "A key-value pair",
+                   "properties": {
+                       "key": {
+                           "type": "string"
+                       },
+                       "value": {
+                           "type": "string"
+                       }
+                   },
+                   "required": [
+                       "key",
+                       "value"
+                   ]
+               }

tests/test_validation.py

@@ -301,7 +301,7 @@ def test_aggregation_metadata_from_form():
         "title": "asdf",
         "subjects": ["Small", "Logan", "VRT"],
         "language": "eng",
-        "additional_metadata": {"key": "value", "another_key": "another_value"},
+        "additional_metadata": [{"key": "key1", "value": "value1"}, {"key": "another key", "value": "another value"}],
         "spatial_coverage": {
             "name": "12232",
             "northlimit": 30.214583003567654,
@@ -348,3 +348,5 @@ def test_aggregation_metadata_from_form():
     agg = GeographicRasterMetadata(**md)
     assert agg.spatial_reference.type == "box"
     assert agg.spatial_coverage.type == "box"
+    assert agg.additional_metadata["key1"] == "value1"
+    assert agg.additional_metadata["another key"] == "another value"