feat(core): add support for NGSI-LD Null and deletedAt Temporal Property

- add support for deletedAt temporal property
- permanently delete attributes when deleted from temporal endpoint
- filter deleted attributes and entities in discovery endpoints
- excluded deleted attributes when building current state of an entity
- handle previous deleted attributes when creating a new one
- do not filter deleted attributes / entities in temporal queries
- add support for NGSI-LD Null in Merge Entity operation
- add support for NGSI-LD Null in Update Attributes operation
- add support for NGSI-LD Null in Partial Attribute Update operation
- add support for Deleted events in event service
- align scope history with new history management
- gdpr: permanently delete entities when cascading deletion of an user
- authz: add option to include deleted entities in get authorized entities endpoint
- add the deleted representation when deleting an entity

build.gradle.kts

@@ -79,6 +79,7 @@ subprojects {
         runtimeOnly("io.micrometer:micrometer-registry-prometheus")
 
         testImplementation("org.springframework.boot:spring-boot-starter-test")
+        testImplementation("org.springframework.boot:spring-boot-testcontainers")
         testImplementation("io.projectreactor:reactor-test")
         testImplementation("com.ninja-squad:springmockk:4.0.2")
         testImplementation("org.springframework.security:spring-security-test")

search-service/config/detekt/baseline.xml

@@ -5,13 +5,13 @@
     <ID>ClassNaming:V0_29_JsonLd_migrationTests.kt$V0_29_JsonLd_migrationTests</ID>
     <ID>ClassNaming:V0_29__JsonLd_migration.kt$V0_29__JsonLd_migration : BaseJavaMigration</ID>
     <ID>ComplexCondition:EntitiesQueryUtils.kt$geoQuery == null &amp;&amp; q.isNullOrEmpty() &amp;&amp; typeSelection.isNullOrEmpty() &amp;&amp; attrs.isEmpty()</ID>
-    <ID>ComplexCondition:EntityQueryService.kt$EntityQueryService$it &amp;&amp; !inverse || !it &amp;&amp; inverse</ID>
     <ID>Filename:V0_29__JsonLd_migration.kt$db.migration.V0_29__JsonLd_migration.kt</ID>
     <ID>LongMethod:AttributeInstanceService.kt$AttributeInstanceService$@Transactional suspend fun create(attributeInstance: AttributeInstance): Either&lt;APIException, Unit&gt;</ID>
     <ID>LongMethod:EnabledAuthorizationServiceTests.kt$EnabledAuthorizationServiceTests$@Test fun `it should return serialized access control entities with other rigths if user is owner`()</ID>
     <ID>LongMethod:EntityAccessControlHandler.kt$EntityAccessControlHandler$@PostMapping("/{subjectId}/attrs", consumes = [MediaType.APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE]) suspend fun addRightsOnEntities( @RequestHeader httpHeaders: HttpHeaders, @PathVariable subjectId: String, @RequestBody requestBody: Mono&lt;String&gt;, @AllowedParameters @RequestParam queryParams: MultiValueMap&lt;String, String&gt; ): ResponseEntity&lt;*&gt;</ID>
     <ID>LongMethod:EntityHandler.kt$EntityHandler$@GetMapping("/{entityId}", produces = [APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE, GEO_JSON_CONTENT_TYPE]) suspend fun getByURI( @RequestHeader httpHeaders: HttpHeaders, @PathVariable entityId: URI, @AllowedParameters( implemented = [ QP.OPTIONS, QP.TYPE, QP.ATTRS, QP.GEOMETRY_PROPERTY, QP.LANG, QP.CONTAINED_BY, QP.JOIN, QP.JOIN_LEVEL, QP.DATASET_ID, ], notImplemented = [QP.FORMAT, QP.PICK, QP.OMIT, QP.ENTITY_MAP, QP.LOCAL, QP.VIA] ) @RequestParam queryParams: MultiValueMap&lt;String, String&gt; ): ResponseEntity&lt;*&gt;</ID>
     <ID>LongMethod:EntityHandler.kt$EntityHandler$@GetMapping(produces = [APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE, GEO_JSON_CONTENT_TYPE]) suspend fun getEntities( @RequestHeader httpHeaders: HttpHeaders, @AllowedParameters( implemented = [ QP.OPTIONS, QP.COUNT, QP.OFFSET, QP.LIMIT, QP.ID, QP.TYPE, QP.ID_PATTERN, QP.ATTRS, QP.Q, QP.GEOMETRY, QP.GEOREL, QP.COORDINATES, QP.GEOPROPERTY, QP.GEOMETRY_PROPERTY, QP.LANG, QP.SCOPEQ, QP.CONTAINED_BY, QP.JOIN, QP.JOIN_LEVEL, QP.DATASET_ID, ], notImplemented = [QP.FORMAT, QP.PICK, QP.OMIT, QP.EXPAND_VALUES, QP.CSF, QP.ENTITY_MAP, QP.LOCAL, QP.VIA] ) @RequestParam queryParams: MultiValueMap&lt;String, String&gt; ): ResponseEntity&lt;*&gt;</ID>
+    <ID>LongMethod:EntityEventService.kt$EntityEventService$private fun publishAttributeChangeEvent( updatedDetails: UpdatedDetails, sub: String?, tenantName: String, entityId: URI, entityTypesAndPayload: Pair&lt;List&lt;ExpandedTerm&gt;, String&gt;, serializedAttribute: Pair&lt;ExpandedTerm, String&gt;, overwrite: Boolean )</ID>
     <ID>LongMethod:LinkedEntityServiceTests.kt$LinkedEntityServiceTests$@Test fun `it should inline entities up to the asked 2nd level`()</ID>
     <ID>LongMethod:PatchAttributeTests.kt$PatchAttributeTests.Companion$@JvmStatic fun mergePatchProvider(): Stream&lt;Arguments&gt;</ID>
     <ID>LongMethod:PatchAttributeTests.kt$PatchAttributeTests.Companion$@JvmStatic fun partialUpdatePatchProvider(): Stream&lt;Arguments&gt;</ID>

search-service/src/main/kotlin/com/egm/stellio/search/authorization/listener/IAMListener.kt

@@ -125,7 +125,7 @@ class IAMListener(
             // (if it no longer exists, it fails because of access rights checks)
             if (searchProperties.onOwnerDeleteCascadeEntities && subjectType == SubjectType.USER) {
                 entityAccessRightsService.getEntitiesIdsOwnedBySubject(sub).getOrNull()?.forEach { entityId ->
-                    entityService.deleteEntity(entityId, sub)
+                    entityService.permanentlyDeleteEntity(entityId, sub)
                 }
                 Unit.right()
             } else Unit.right()

search-service/src/main/kotlin/com/egm/stellio/search/authorization/model/EntityAccessRights.kt

@@ -6,6 +6,7 @@ import com.egm.stellio.shared.model.addNonReifiedProperty
 import com.egm.stellio.shared.model.addSubAttribute
 import com.egm.stellio.shared.util.AccessRight
 import com.egm.stellio.shared.util.AuthContextModel
+import com.egm.stellio.shared.util.AuthContextModel.AUTH_PROP_IS_DELETED
 import com.egm.stellio.shared.util.AuthContextModel.AUTH_PROP_RIGHT
 import com.egm.stellio.shared.util.AuthContextModel.AUTH_PROP_SAP
 import com.egm.stellio.shared.util.AuthContextModel.AUTH_PROP_SUBJECT_INFO
@@ -27,6 +28,7 @@ import java.net.URI
 data class EntityAccessRights(
     val id: URI,
     val types: List<ExpandedTerm>,
+    val isDeleted: Boolean = false,
     // right the current user has on the entity
     val right: AccessRight,
     val specificAccessPolicy: AuthContextModel.SpecificAccessPolicy? = null,
@@ -55,6 +57,8 @@ data class EntityAccessRights(
 
         resultEntity[JSONLD_ID] = id.toString()
         resultEntity[JSONLD_TYPE] = types
+        if (isDeleted)
+            resultEntity[AUTH_PROP_IS_DELETED] = buildExpandedPropertyValue(true)
         resultEntity[AUTH_PROP_RIGHT] = buildExpandedPropertyValue(right.attributeName)
 
         specificAccessPolicy?.run {

search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/AuthorizationService.kt

@@ -23,6 +23,7 @@ interface AuthorizationService {
 
     suspend fun getAuthorizedEntities(
         entitiesQuery: EntitiesQueryFromGet,
+        includeDeleted: Boolean,
         contexts: List<String>,
         sub: Option<Sub>
     ): Either<APIException, Pair<Int, List<ExpandedEntity>>>

search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/DisabledAuthorizationService.kt

@@ -41,6 +41,7 @@ class DisabledAuthorizationService : AuthorizationService {
 
     override suspend fun getAuthorizedEntities(
         entitiesQuery: EntitiesQueryFromGet,
+        includeDeleted: Boolean,
         contexts: List<String>,
         sub: Option<Sub>
     ): Either<APIException, Pair<Int, List<ExpandedEntity>>> = Pair(-1, emptyList<ExpandedEntity>()).right()

search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/EnabledAuthorizationService.kt

@@ -108,16 +108,16 @@ class EnabledAuthorizationService(
 
     override suspend fun getAuthorizedEntities(
         entitiesQuery: EntitiesQueryFromGet,
+        includeDeleted: Boolean,
         contexts: List<String>,
         sub: Option<Sub>
     ): Either<APIException, Pair<Int, List<ExpandedEntity>>> = either {
         val accessRights = entitiesQuery.attrs.mapNotNull { AccessRight.forExpandedAttributeName(it).getOrNull() }
         val entitiesAccessRights = entityAccessRightsService.getSubjectAccessRights(
             sub,
             accessRights,
-            entitiesQuery.typeSelection,
-            entitiesQuery.ids,
-            entitiesQuery.paginationQuery
+            entitiesQuery,
+            includeDeleted
         ).bind()
 
         // for each entity user is admin or creator of, retrieve the full details of rights other users have on it
@@ -148,7 +148,8 @@ class EnabledAuthorizationService(
             sub,
             accessRights,
             entitiesQuery.typeSelection,
-            entitiesQuery.ids
+            entitiesQuery.ids,
+            includeDeleted
         ).bind()
 
         Pair(count, entitiesAccessControlWithSubjectRights)

search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/EntityAccessRightsService.kt

@@ -18,13 +18,13 @@ import com.egm.stellio.search.common.util.toJsonString
 import com.egm.stellio.search.common.util.toList
 import com.egm.stellio.search.common.util.toOptionalEnum
 import com.egm.stellio.search.common.util.toUri
+import com.egm.stellio.search.entity.model.EntitiesQueryFromGet
 import com.egm.stellio.shared.config.ApplicationProperties
 import com.egm.stellio.shared.model.APIException
 import com.egm.stellio.shared.model.AccessDeniedException
 import com.egm.stellio.shared.model.EntityTypeSelection
 import com.egm.stellio.shared.model.NgsiLdAttribute
 import com.egm.stellio.shared.model.ResourceNotFoundException
-import com.egm.stellio.shared.queryparameter.PaginationQuery
 import com.egm.stellio.shared.util.AccessRight
 import com.egm.stellio.shared.util.AccessRight.CAN_ADMIN
 import com.egm.stellio.shared.util.AccessRight.CAN_READ
@@ -220,30 +220,32 @@ class EntityAccessRightsService(
     suspend fun getSubjectAccessRights(
         sub: Option<Sub>,
         accessRights: List<AccessRight>,
-        type: EntityTypeSelection? = null,
-        ids: Set<URI>? = null,
-        paginationQuery: PaginationQuery,
+        entitiesQuery: EntitiesQueryFromGet,
+        includeDeleted: Boolean = false
     ): Either<APIException, List<EntityAccessRights>> = either {
+        val ids = entitiesQuery.ids
+        val typeSelection = entitiesQuery.typeSelection
         val subjectUuids = subjectReferentialService.getSubjectAndGroupsUUID(sub).bind()
         val isStellioAdmin = subjectReferentialService.hasStellioAdminRole(subjectUuids).bind()
 
         databaseClient
             .sql(
                 """
-                SELECT ep.entity_id, ep.types, ear.access_right, ep.specific_access_policy
+                SELECT ep.entity_id, ep.types, ear.access_right, ep.specific_access_policy, ep.deleted_at
                 FROM entity_access_rights ear
                 LEFT JOIN entity_payload ep ON ear.entity_id = ep.entity_id
                 WHERE ${if (isStellioAdmin) "1 = 1" else "subject_id IN (:subject_uuids)" }
                 ${if (accessRights.isNotEmpty()) " AND access_right IN (:access_rights)" else ""}
-                ${if (!type.isNullOrEmpty()) " AND (${buildTypeQuery(type)})" else ""}
-                ${if (!ids.isNullOrEmpty()) " AND ear.entity_id IN (:entities_ids)" else ""}
+                ${if (!typeSelection.isNullOrEmpty()) " AND (${buildTypeQuery(typeSelection)})" else ""}
+                ${if (ids.isNotEmpty()) " AND ear.entity_id IN (:entities_ids)" else ""}
+                ${if (!includeDeleted) " AND deleted_at IS NULL" else ""}
                 ORDER BY entity_id
                 LIMIT :limit
                 OFFSET :offset;
                 """.trimIndent()
             )
-            .bind("limit", paginationQuery.limit)
-            .bind("offset", paginationQuery.offset)
+            .bind("limit", entitiesQuery.paginationQuery.limit)
+            .bind("offset", entitiesQuery.paginationQuery.offset)
             .let {
                 if (!isStellioAdmin)
                     it.bind("subject_uuids", subjectUuids)
@@ -255,7 +257,7 @@ class EntityAccessRightsService(
                 else it
             }
             .let {
-                if (!ids.isNullOrEmpty())
+                if (ids.isNotEmpty())
                     it.bind("entities_ids", ids)
                 else it
             }
@@ -268,6 +270,7 @@ class EntityAccessRightsService(
                 EntityAccessRights(
                     ear.id,
                     ear.types,
+                    ear.isDeleted,
                     entityAccessRights.maxOf { it.right },
                     ear.specificAccessPolicy
                 )
@@ -278,7 +281,8 @@ class EntityAccessRightsService(
         sub: Option<Sub>,
         accessRights: List<AccessRight>,
         type: EntityTypeSelection? = null,
-        ids: Set<URI>? = null
+        ids: Set<URI>? = null,
+        includeDeleted: Boolean = false
     ): Either<APIException, Int> = either {
         val subjectUuids = subjectReferentialService.getSubjectAndGroupsUUID(sub).bind()
         val isStellioAdmin = subjectReferentialService.hasStellioAdminRole(subjectUuids).bind()
@@ -293,6 +297,7 @@ class EntityAccessRightsService(
                 ${if (accessRights.isNotEmpty()) " AND access_right IN (:access_rights)" else ""}
                 ${if (!type.isNullOrEmpty()) " AND (${buildTypeQuery(type)})" else ""}
                 ${if (!ids.isNullOrEmpty()) " AND ear.entity_id IN (:entities_ids)" else ""}
+                ${if (!includeDeleted) " AND deleted_at IS NULL" else ""}
                 """.trimIndent()
             )
             .let {
@@ -443,6 +448,7 @@ class EntityAccessRightsService(
         return EntityAccessRights(
             id = toUri(row["entity_id"]),
             types = toList(row["types"]),
+            isDeleted = row["deleted_at"] != null,
             right = accessRight,
             specificAccessPolicy = toOptionalEnum<SpecificAccessPolicy>(row["specific_access_policy"])
         )

search-service/src/main/kotlin/com/egm/stellio/search/authorization/web/EntityAccessControlHandler.kt

@@ -19,6 +19,7 @@ import com.egm.stellio.shared.model.toNgsiLdAttribute
 import com.egm.stellio.shared.model.toNgsiLdAttributes
 import com.egm.stellio.shared.queryparameter.AllowedParameters
 import com.egm.stellio.shared.queryparameter.QP
+import com.egm.stellio.shared.queryparameter.QueryParameter
 import com.egm.stellio.shared.util.AccessRight
 import com.egm.stellio.shared.util.AuthContextModel.ALL_ASSIGNABLE_IAM_RIGHTS
 import com.egm.stellio.shared.util.AuthContextModel.ALL_IAM_RIGHTS
@@ -70,10 +71,11 @@ class EntityAccessControlHandler(
     @GetMapping("/entities", produces = [MediaType.APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE])
     suspend fun getAuthorizedEntities(
         @RequestHeader httpHeaders: HttpHeaders,
-        @AllowedParameters(implemented = [QP.ID, QP.TYPE, QP.ATTRS, QP.COUNT, QP.OFFSET, QP.LIMIT])
+        @AllowedParameters(implemented = [QP.ID, QP.TYPE, QP.ATTRS, QP.COUNT, QP.OFFSET, QP.LIMIT, QP.INCLUDE_DELETED])
         @RequestParam queryParams: MultiValueMap<String, String>
     ): ResponseEntity<*> = either {
         val sub = getSubFromSecurityContext()
+        val includeDeleted = queryParams.getFirst(QueryParameter.INCLUDE_DELETED.key)?.toBoolean() == true
 
         val contexts = getAuthzContextFromLinkHeaderOrDefault(httpHeaders, applicationProperties.contexts).bind()
         val mediaType = getApplicableMediaType(httpHeaders).bind()
@@ -91,6 +93,7 @@ class EntityAccessControlHandler(
 
         val (count, entities) = authorizationService.getAuthorizedEntities(
             entitiesQuery,
+            includeDeleted,
             contexts,
             sub
         ).bind()

search-service/src/main/kotlin/com/egm/stellio/search/discovery/service/AttributeService.kt

@@ -30,6 +30,7 @@ class AttributeService(
             """
             SELECT DISTINCT(attribute_name)
             FROM temporal_entity_attribute
+            WHERE deleted_at IS NULL
             ORDER BY attribute_name
             """.trimIndent()
         ).allToMappedList { rowToAttributeNames(it) }
@@ -42,7 +43,10 @@ class AttributeService(
             """
             SELECT types, attribute_name
             FROM entity_payload
-            JOIN temporal_entity_attribute ON entity_payload.entity_id = temporal_entity_attribute.entity_id
+            JOIN temporal_entity_attribute 
+                ON entity_payload.entity_id = temporal_entity_attribute.entity_id
+                AND temporal_entity_attribute.deleted_at IS NULL
+            WHERE entity_payload.deleted_at IS NULL
             ORDER BY attribute_name
             """.trimIndent()
         ).allToMappedList { rowToAttributeDetails(it) }.flatten().groupBy({ it.second }, { it.first }).toList()
@@ -65,11 +69,14 @@ class AttributeService(
             WITH entities AS (
                 SELECT entity_id, attribute_name, attribute_type
                 FROM temporal_entity_attribute 
-                WHERE  attribute_name = :attribute_name
+                WHERE attribute_name = :attribute_name
+                AND deleted_at IS NULL
             )    
             SELECT attribute_name, attribute_type, types, count(distinct(attribute_name)) as attribute_count
             FROM entity_payload
-            JOIN entities ON entity_payload.entity_id = entities.entity_id
+            JOIN entities 
+                ON entity_payload.entity_id = entities.entity_id
+                AND entity_payload.deleted_at IS NULL
             GROUP BY types, attribute_name, attribute_type
             """.trimIndent()
         )

search-service/src/main/kotlin/com/egm/stellio/search/discovery/service/EntityTypeService.kt

@@ -29,6 +29,7 @@ class EntityTypeService(
             """
             SELECT DISTINCT(unnest(types)) as type
             FROM entity_payload
+            WHERE deleted_at IS NULL
             ORDER BY type
             """.trimIndent()
         ).allToMappedList { rowToType(it) }
@@ -41,7 +42,10 @@ class EntityTypeService(
             """
             SELECT unnest(types) as type, attribute_name
             FROM entity_payload
-            JOIN temporal_entity_attribute ON entity_payload.entity_id = temporal_entity_attribute.entity_id
+            JOIN temporal_entity_attribute 
+                ON entity_payload.entity_id = temporal_entity_attribute.entity_id
+                AND temporal_entity_attribute.deleted_at IS NULL
+            WHERE entity_payload.deleted_at IS NULL
             ORDER BY type
             """.trimIndent()
         ).allToMappedList { rowToEntityType(it) }.groupBy({ it.first }, { it.second }).toList()
@@ -65,10 +69,12 @@ class EntityTypeService(
                 SELECT entity_id
                 FROM entity_payload 
                 WHERE :type_name = any (types)
+                AND deleted_at IS NULL
             )    
             SELECT attribute_name, attribute_type, (select count(entity_id) from entities) as entity_count
             FROM temporal_entity_attribute
             WHERE entity_id IN (SELECT entity_id FROM entities)
+            AND deleted_at IS NULL
             GROUP BY attribute_name, attribute_type
             """.trimIndent()
         )