feat(core): add support for LanguageProperty (#1120)

* feat: add support for LanguageProperty in entities * feat: add support for LanguageProperty in discovery endpoints * feat: temporalValues representation of a LanguageProperty * feat: support for NGSI-LD Language Filter * feat: handle @none for default values * feat: lang filter in subcriptions * feat: merge patch for LanguageProperty (and other attributes types)
stellio-hub · Apr 25, 2024 · 0cf25a9 · 0cf25a9
1 parent 3017dc5
commit 0cf25a9
Show file tree

Hide file tree

Showing 39 changed files with 1,359 additions and 199 deletions.
diff --git a/search-service/config/detekt/baseline.xml b/search-service/config/detekt/baseline.xml
@@ -10,6 +10,8 @@
     <ID>LongMethod:AttributeInstanceService.kt$AttributeInstanceService$@Transactional suspend fun create(attributeInstance: AttributeInstance): Either&lt;APIException, Unit&gt;</ID>
     <ID>LongMethod:EntityAccessControlHandler.kt$EntityAccessControlHandler$@PostMapping("/{subjectId}/attrs", consumes = [MediaType.APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE]) suspend fun addRightsOnEntities( @RequestHeader httpHeaders: HttpHeaders, @PathVariable subjectId: String, @RequestBody requestBody: Mono&lt;String&gt; ): ResponseEntity&lt;*&gt;</ID>
     <ID>LongMethod:EntityOperationHandlerTests.kt$EntityOperationHandlerTests$@Test fun `create batch entity should return a 207 when some entities already exist`()</ID>
+    <ID>LongMethod:PatchAttributeTests.kt$PatchAttributeTests.Companion$@JvmStatic fun mergePatchProvider(): Stream&lt;Arguments&gt;</ID>
+    <ID>LongMethod:PatchAttributeTests.kt$PatchAttributeTests.Companion$@JvmStatic fun partialUpdatePatchProvider(): Stream&lt;Arguments&gt;</ID>
     <ID>LongMethod:QueryServiceTests.kt$QueryServiceTests$@Test fun `it should query temporal entities as requested by query params`()</ID>
     <ID>LongMethod:QueryServiceTests.kt$QueryServiceTests$@Test fun `it should return an empty list for an attribute if it has no temporal values`()</ID>
     <ID>LongMethod:TemporalEntityBuilderTests.kt$TemporalEntityBuilderTests$@Test fun `it should return a temporal entity with values aggregated`()</ID>

diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/model/EntityTypeInfo.kt b/search-service/src/main/kotlin/com/egm/stellio/search/model/EntityTypeInfo.kt
@@ -22,7 +22,8 @@ enum class AttributeType(val key: String) {
     Property("Property"),
     Relationship("Relationship"),
     GeoProperty("GeoProperty"),
-    JsonProperty("JsonProperty");
+    JsonProperty("JsonProperty"),
+    LanguageProperty("LanguageProperty");
 
     companion object {
         fun forKey(key: String): AttributeType =

diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/model/Query.kt b/search-service/src/main/kotlin/com/egm/stellio/search/model/Query.kt
@@ -22,7 +22,8 @@ data class Query private constructor(
     val q: String? = null,
     val geoQ: UnparsedGeoQuery? = null,
     val temporalQ: UnparsedTemporalQuery? = null,
-    val scopeQ: String? = null
+    val scopeQ: String? = null,
+    val lang: String? = null,
 ) {
     companion object {
         operator fun invoke(queryBody: String): Either<APIException, Query> = either {

diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalEntityAttribute.kt b/search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalEntityAttribute.kt
@@ -5,6 +5,8 @@ import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_GEOPROPERTY_TYPE
 import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_GEOPROPERTY_VALUES
 import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_JSONPROPERTY_TYPE
 import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_JSONPROPERTY_VALUES
+import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_LANGUAGEPROPERTY_TYPE
+import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_LANGUAGEPROPERTY_VALUES
 import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_PROPERTY_TYPE
 import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_PROPERTY_VALUES
 import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_RELATIONSHIP_OBJECTS
@@ -45,14 +47,16 @@ data class TemporalEntityAttribute(
         Property,
         Relationship,
         GeoProperty,
-        JsonProperty;
+        JsonProperty,
+        LanguageProperty;
 
         fun toExpandedName(): String =
             when (this) {
                 Property -> NGSILD_PROPERTY_TYPE.uri
                 Relationship -> NGSILD_RELATIONSHIP_TYPE.uri
                 GeoProperty -> NGSILD_GEOPROPERTY_TYPE.uri
                 JsonProperty -> NGSILD_JSONPROPERTY_TYPE.uri
+                LanguageProperty -> NGSILD_LANGUAGEPROPERTY_TYPE.uri
             }
 
         /**
@@ -64,6 +68,7 @@ data class TemporalEntityAttribute(
                 Relationship -> NGSILD_RELATIONSHIP_OBJECTS
                 GeoProperty -> NGSILD_GEOPROPERTY_VALUES
                 JsonProperty -> NGSILD_JSONPROPERTY_VALUES
+                LanguageProperty -> NGSILD_LANGUAGEPROPERTY_VALUES
             }
     }
 }
diff --git a/...-service/src/main/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeService.kt b/...-service/src/main/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeService.kt
@@ -14,15 +14,14 @@ import com.egm.stellio.shared.util.*
 import com.egm.stellio.shared.util.AttributeType
 import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE
 import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_JSONPROPERTY_VALUE
+import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_LANGUAGEPROPERTY_VALUE
 import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_OBSERVED_AT_PROPERTY
 import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_PREFIX
 import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_RELATIONSHIP_OBJECT
 import com.egm.stellio.shared.util.JsonLdUtils.buildNonReifiedTemporalValue
 import com.egm.stellio.shared.util.JsonLdUtils.expandJsonLdEntity
 import com.egm.stellio.shared.util.JsonUtils.serializeObject
-import com.savvasdalkitsis.jsonmerger.JsonMerger
 import io.r2dbc.postgresql.codec.Json
-import org.json.JSONObject
 import org.slf4j.LoggerFactory
 import org.springframework.r2dbc.core.DatabaseClient
 import org.springframework.r2dbc.core.bind
@@ -270,9 +269,10 @@ class TemporalEntityAttributeService(
             attributeMetadata,
             observedAt
         )
-        val (jsonTargetObject, updatedAttributeInstance) = mergeAttributePayload(tea, processedAttributePayload)
+        val (jsonTargetObject, updatedAttributeInstance) =
+            mergePatch(tea.payload.toExpandedAttributeInstance(), processedAttributePayload)
         val value = getValueFromPartialAttributePayload(tea, updatedAttributeInstance)
-        updateOnUpdate(tea.id, processedAttributeMetadata.valueType, mergedAt, jsonTargetObject.toString()).bind()
+        updateOnUpdate(tea.id, processedAttributeMetadata.valueType, mergedAt, jsonTargetObject).bind()
 
         val attributeInstance =
             createContextualAttributeInstance(tea, updatedAttributeInstance, value, mergedAt, sub)
@@ -657,10 +657,11 @@ class TemporalEntityAttributeService(
                         BadRequestDataException("The type of the attribute has to be the same as the existing one")
                     }
                 }
-                val (jsonTargetObject, updatedAttributeInstance) = mergeAttributePayload(tea, attributeValues)
+                val (jsonTargetObject, updatedAttributeInstance) =
+                    partialUpdatePatch(tea.payload.toExpandedAttributeInstance(), attributeValues)
                 val value = getValueFromPartialAttributePayload(tea, updatedAttributeInstance)
                 val attributeValueType = guessAttributeValueType(tea.attributeType, attributeValues)
-                updateOnUpdate(tea.id, attributeValueType, modifiedAt, jsonTargetObject.toString()).bind()
+                updateOnUpdate(tea.id, attributeValueType, modifiedAt, jsonTargetObject).bind()
 
                 // then update attribute instance
                 val attributeInstance = createContextualAttributeInstance(
@@ -863,33 +864,14 @@ class TemporalEntityAttributeService(
                     null,
                     null
                 )
+            TemporalEntityAttribute.AttributeType.LanguageProperty ->
+                Triple(
+                    serializeObject(attributePayload.getMemberValue(NGSILD_LANGUAGEPROPERTY_VALUE)!!),
+                    null,
+                    null
+                )
         }
 
-    suspend fun mergeAttributePayload(
-        tea: TemporalEntityAttribute,
-        expandedAttributeInstance: ExpandedAttributeInstance
-    ): Pair<JSONObject, ExpandedAttributeInstance> {
-        val jsonSourceObject = JSONObject(tea.payload.asString())
-        val jsonUpdateObject = JSONObject(expandedAttributeInstance)
-        // if the attribute is a JsonProperty, preserve its JSON value to avoid it being merged
-        // (the whole JSON value shall be replaced)
-        val preservedJsonValue = if (tea.attributeType == TemporalEntityAttribute.AttributeType.JsonProperty)
-            expandedAttributeInstance[NGSILD_JSONPROPERTY_VALUE]
-        else null
-        val jsonMerger = JsonMerger(
-            arrayMergeMode = JsonMerger.ArrayMergeMode.REPLACE_ARRAY,
-            objectMergeMode = JsonMerger.ObjectMergeMode.MERGE_OBJECT
-        )
-        val jsonTargetObject = jsonMerger.merge(jsonSourceObject, jsonUpdateObject)
-            .let {
-                if (preservedJsonValue != null)
-                    it.put(NGSILD_JSONPROPERTY_VALUE, preservedJsonValue)
-                else it
-            }
-        val updatedAttributeInstance = jsonTargetObject.toMap() as ExpandedAttributeInstance
-        return Pair(jsonTargetObject, updatedAttributeInstance)
-    }
-
     private fun createContextualAttributeInstance(
         tea: TemporalEntityAttribute,
         expandedAttributeInstance: ExpandedAttributeInstance,
@@ -926,16 +908,15 @@ class TemporalEntityAttributeService(
         attributePayload: ExpandedAttributeInstance,
         attributeMetadata: AttributeMetadata,
         observedAt: ZonedDateTime?
-    ): Pair<ExpandedAttributeInstance, AttributeMetadata> {
-        return if (
+    ): Pair<ExpandedAttributeInstance, AttributeMetadata> =
+        if (
             observedAt != null &&
             tea.payload.deserializeAsMap().containsKey(NGSILD_OBSERVED_AT_PROPERTY) &&
             !attributePayload.containsKey(NGSILD_OBSERVED_AT_PROPERTY)
-        ) {
+        )
             Pair(
                 attributePayload.plus(NGSILD_OBSERVED_AT_PROPERTY to buildNonReifiedTemporalValue(observedAt)),
                 attributeMetadata.copy(observedAt = observedAt)
             )
-        } else Pair(attributePayload, attributeMetadata)
-    }
+        else Pair(attributePayload, attributeMetadata)
 }
diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/util/AttributeInstanceUtils.kt b/search-service/src/main/kotlin/com/egm/stellio/search/util/AttributeInstanceUtils.kt
@@ -8,8 +8,15 @@ import com.egm.stellio.search.model.AttributeMetadata
 import com.egm.stellio.search.model.TemporalEntityAttribute
 import com.egm.stellio.search.model.TemporalEntityAttribute.AttributeValueType
 import com.egm.stellio.shared.model.*
+import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_LANGUAGE
+import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_JSONPROPERTY_VALUE
+import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_LANGUAGEPROPERTY_VALUE
+import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_PROPERTY_VALUE
 import com.egm.stellio.shared.util.JsonLdUtils.logger
+import com.egm.stellio.shared.util.JsonUtils.deserializeAsMap
 import com.egm.stellio.shared.util.JsonUtils.serializeObject
+import com.savvasdalkitsis.jsonmerger.JsonMerger
+import io.r2dbc.postgresql.codec.Json
 import java.time.LocalDate
 import java.time.LocalTime
 import java.time.ZonedDateTime
@@ -65,6 +72,12 @@ fun NgsiLdAttributeInstance.toTemporalAttributeMetadata(): Either<APIException,
                 AttributeValueType.JSON,
                 Triple(serializeObject(this.json), null, null)
             )
+        is NgsiLdLanguagePropertyInstance ->
+            Triple(
+                TemporalEntityAttribute.AttributeType.LanguageProperty,
+                AttributeValueType.ARRAY,
+                Triple(serializeObject(this.languageMap), null, null)
+            )
     }
     if (attributeValue == Triple(null, null, null)) {
         logger.warn("Unable to get a value from attribute: $this")
@@ -92,6 +105,7 @@ fun guessAttributeValueType(
         TemporalEntityAttribute.AttributeType.Relationship -> AttributeValueType.URI
         TemporalEntityAttribute.AttributeType.GeoProperty -> AttributeValueType.GEOMETRY
         TemporalEntityAttribute.AttributeType.JsonProperty -> AttributeValueType.JSON
+        TemporalEntityAttribute.AttributeType.LanguageProperty -> AttributeValueType.ARRAY
     }
 
 fun guessPropertyValueType(
@@ -114,3 +128,55 @@ fun guessPropertyValueType(
         is LocalTime -> Pair(AttributeValueType.TIME, Triple(value.toString(), null, null))
         else -> Pair(AttributeValueType.STRING, Triple(value.toString(), null, null))
     }
+
+fun Json.toExpandedAttributeInstance(): ExpandedAttributeInstance =
+    this.deserializeAsMap() as ExpandedAttributeInstance
+
+fun partialUpdatePatch(
+    source: ExpandedAttributeInstance,
+    update: ExpandedAttributeInstance
+): Pair<String, ExpandedAttributeInstance> {
+    val target = source.plus(update)
+    return Pair(serializeObject(target), target)
+}
+
+fun mergePatch(
+    source: ExpandedAttributeInstance,
+    update: ExpandedAttributeInstance
+): Pair<String, ExpandedAttributeInstance> {
+    val target = source.toMutableMap()
+    update.forEach { (attrName, attrValue) ->
+        if (!source.containsKey(attrName)) {
+            target[attrName] = attrValue
+        } else if (listOf(NGSILD_JSONPROPERTY_VALUE, NGSILD_PROPERTY_VALUE).contains(attrName)) {
+            if (attrValue.size > 1) {
+                // a Property holding an array of value or a JsonPropery holding an array of JSON objects
+                // cannot be safely merged patch, so copy the whole value from the update
+                target[attrName] = attrValue
+            } else {
+                target[attrName] = listOf(
+                    JsonMerger().merge(
+                        serializeObject(source[attrName]!![0]),
+                        serializeObject(attrValue[0])
+                    ).deserializeAsMap()
+                )
+            }
+        } else if (listOf(NGSILD_LANGUAGEPROPERTY_VALUE).contains(attrName)) {
+            val sourceLangEntries = source[attrName] as List<Map<String, String>>
+            val targetLangEntries = sourceLangEntries.toMutableList()
+            (attrValue as List<Map<String, String>>).forEach { langEntry ->
+                // remove any previously existing entry for this language
+                targetLangEntries.removeIf {
+                    it[JSONLD_LANGUAGE] == langEntry[JSONLD_LANGUAGE]
+                }
+                targetLangEntries.add(langEntry)
+            }
+
+            target[attrName] = targetLangEntries
+        } else {
+            target[attrName] = attrValue
+        }
+    }
+
+    return Pair(serializeObject(target), target)
+}
diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/util/EntitiesQueryUtils.kt b/search-service/src/main/kotlin/com/egm/stellio/search/util/EntitiesQueryUtils.kt
@@ -63,16 +63,6 @@ fun EntitiesQuery.validateMinimalQueryEntitiesParameters(): Either<APIException,
     this@validateMinimalQueryEntitiesParameters
 }
 
-fun composeEntitiesQueryFromPostRequest(
-    defaultPagination: ApplicationProperties.Pagination,
-    requestBody: String,
-    requestParams: MultiValueMap<String, String>,
-    contexts: List<String>
-): Either<APIException, EntitiesQuery> = either {
-    val query = Query(requestBody).bind()
-    composeEntitiesQueryFromPostRequest(defaultPagination, query, requestParams, contexts).bind()
-}
-
 fun composeEntitiesQueryFromPostRequest(
     defaultPagination: ApplicationProperties.Pagination,
     query: Query,
@@ -152,11 +142,10 @@ fun composeTemporalEntitiesQuery(
 
 fun composeTemporalEntitiesQueryFromPostRequest(
     defaultPagination: ApplicationProperties.Pagination,
-    requestBody: String,
+    query: Query,
     requestParams: MultiValueMap<String, String>,
     contexts: List<String>
 ): Either<APIException, TemporalEntitiesQuery> = either {
-    val query = Query(requestBody).bind()
     val entitiesQuery = composeEntitiesQueryFromPostRequest(
         defaultPagination,
         query,

diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/util/TemporalEntityBuilder.kt b/search-service/src/main/kotlin/com/egm/stellio/search/util/TemporalEntityBuilder.kt
@@ -12,6 +12,7 @@ import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_VALUE_TERM
 import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DATASET_ID_PROPERTY
 import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_GEOPROPERTY_TYPE
 import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_JSONPROPERTY_VALUE
+import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_LANGUAGEPROPERTY_VALUE
 import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_PREFIX
 import com.egm.stellio.shared.util.JsonLdUtils.buildExpandedPropertyValue
 import com.egm.stellio.shared.util.JsonLdUtils.buildExpandedTemporalValue
@@ -113,8 +114,8 @@ object TemporalEntityBuilder {
      */
     private fun buildAttributesSimplifiedRepresentation(
         attributeAndResultsMap: TemporalEntityAttributeInstancesResult
-    ): Map<TemporalEntityAttribute, SimplifiedTemporalAttribute> {
-        return attributeAndResultsMap.mapValues {
+    ): Map<TemporalEntityAttribute, SimplifiedTemporalAttribute> =
+        attributeAndResultsMap.mapValues {
             val attributeInstance = mutableMapOf<String, Any>(
                 JSONLD_TYPE to listOf(it.key.attributeType.toExpandedName())
             )
@@ -142,6 +143,14 @@ object TemporalEntityBuilder {
                             ),
                             mapOf(JSONLD_VALUE to attributeInstanceResult.time)
                         )
+                    } else if (it.key.attributeType == TemporalEntityAttribute.AttributeType.LanguageProperty) {
+                        listOf(
+                            mapOf(
+                                NGSILD_LANGUAGEPROPERTY_VALUE to
+                                    deserializeListOfObjects(attributeInstanceResult.value as String)
+                            ),
+                            mapOf(JSONLD_VALUE to attributeInstanceResult.time)
+                        )
                     } else {
                         listOf(
                             mapOf(JSONLD_VALUE to attributeInstanceResult.value),
@@ -151,7 +160,6 @@ object TemporalEntityBuilder {
                 }
             attributeInstance.toMap()
         }
-    }
 
     /**
      * Creates the aggregated representation for each temporal entity attribute in the input map.

diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/web/EntityOperationHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/web/EntityOperationHandler.kt
@@ -3,6 +3,7 @@ package com.egm.stellio.search.web
 import arrow.core.*
 import arrow.core.raise.either
 import com.egm.stellio.search.authorization.AuthorizationService
+import com.egm.stellio.search.model.Query
 import com.egm.stellio.search.service.EntityEventService
 import com.egm.stellio.search.service.EntityOperationService
 import com.egm.stellio.search.service.EntityPayloadService
@@ -272,10 +273,11 @@ class EntityOperationHandler(
         val sub = getSubFromSecurityContext()
         val contexts = getContextFromLinkHeaderOrDefault(httpHeaders).bind()
         val mediaType = getApplicableMediaType(httpHeaders).bind()
+        val query = Query(requestBody.awaitFirst()).bind()
 
         val entitiesQuery = composeEntitiesQueryFromPostRequest(
             applicationProperties.pagination,
-            requestBody.awaitFirst(),
+            query,
             params,
             contexts
         ).bind()
@@ -289,6 +291,8 @@ class EntityOperationHandler(
         val compactedEntities = compactEntities(filteredEntities, contexts)
 
         val ngsiLdDataRepresentation = parseRepresentations(params, mediaType)
+            .copy(languageFilter = query.lang)
+
         buildQueryResponse(
             compactedEntities.toFinalRepresentation(ngsiLdDataRepresentation),
             count,

diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/web/TemporalEntityOperationsHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/web/TemporalEntityOperationsHandler.kt
@@ -2,6 +2,7 @@ package com.egm.stellio.search.web
 
 import arrow.core.raise.either
 import com.egm.stellio.search.authorization.AuthorizationService
+import com.egm.stellio.search.model.Query
 import com.egm.stellio.search.service.QueryService
 import com.egm.stellio.search.util.composeTemporalEntitiesQueryFromPostRequest
 import com.egm.stellio.shared.config.ApplicationProperties
@@ -36,11 +37,12 @@ class TemporalEntityOperationsHandler(
         val sub = getSubFromSecurityContext()
         val contexts = getContextFromLinkHeaderOrDefault(httpHeaders).bind()
         val mediaType = getApplicableMediaType(httpHeaders).bind()
+        val query = Query(requestBody.awaitFirst()).bind()
 
         val temporalEntitiesQuery =
             composeTemporalEntitiesQueryFromPostRequest(
                 applicationProperties.pagination,
-                requestBody.awaitFirst(),
+                query,
                 params,
                 contexts
             ).bind()
@@ -55,6 +57,7 @@ class TemporalEntityOperationsHandler(
         val compactedEntities = compactEntities(temporalEntities, contexts)
 
         val ngsiLdDataRepresentation = parseRepresentations(params, mediaType)
+            .copy(languageFilter = query.lang)
 
         buildQueryResponse(
             compactedEntities.toFinalRepresentation(ngsiLdDataRepresentation),