diff --git a/build.gradle.kts b/build.gradle.kts index 5b1074a5e..4436971f4 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -79,6 +79,7 @@ subprojects { runtimeOnly("io.micrometer:micrometer-registry-prometheus") testImplementation("org.springframework.boot:spring-boot-starter-test") + testImplementation("org.springframework.boot:spring-boot-testcontainers") testImplementation("io.projectreactor:reactor-test") testImplementation("com.ninja-squad:springmockk:4.0.2") testImplementation("org.springframework.security:spring-security-test") diff --git a/search-service/config/detekt/baseline.xml b/search-service/config/detekt/baseline.xml index 7f380d5b4..b2b802489 100644 --- a/search-service/config/detekt/baseline.xml +++ b/search-service/config/detekt/baseline.xml @@ -5,13 +5,13 @@ ClassNaming:V0_29_JsonLd_migrationTests.kt$V0_29_JsonLd_migrationTests ClassNaming:V0_29__JsonLd_migration.kt$V0_29__JsonLd_migration : BaseJavaMigration ComplexCondition:EntitiesQueryUtils.kt$geoQuery == null && q.isNullOrEmpty() && typeSelection.isNullOrEmpty() && attrs.isEmpty() - ComplexCondition:EntityQueryService.kt$EntityQueryService$it && !inverse || !it && inverse Filename:V0_29__JsonLd_migration.kt$db.migration.V0_29__JsonLd_migration.kt LongMethod:AttributeInstanceService.kt$AttributeInstanceService$@Transactional suspend fun create(attributeInstance: AttributeInstance): Either<APIException, Unit> LongMethod:EnabledAuthorizationServiceTests.kt$EnabledAuthorizationServiceTests$@Test fun `it should return serialized access control entities with other rigths if user is owner`() LongMethod:EntityAccessControlHandler.kt$EntityAccessControlHandler$@PostMapping("/{subjectId}/attrs", consumes = [MediaType.APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE]) suspend fun addRightsOnEntities( @RequestHeader httpHeaders: HttpHeaders, @PathVariable subjectId: String, @RequestBody requestBody: Mono<String>, @AllowedParameters @RequestParam queryParams: MultiValueMap<String, String> ): ResponseEntity<*> LongMethod:EntityHandler.kt$EntityHandler$@GetMapping("/{entityId}", produces = [APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE, GEO_JSON_CONTENT_TYPE]) suspend fun getByURI( @RequestHeader httpHeaders: HttpHeaders, @PathVariable entityId: URI, @AllowedParameters( implemented = [ QP.OPTIONS, QP.TYPE, QP.ATTRS, QP.GEOMETRY_PROPERTY, QP.LANG, QP.CONTAINED_BY, QP.JOIN, QP.JOIN_LEVEL, QP.DATASET_ID, ], notImplemented = [QP.FORMAT, QP.PICK, QP.OMIT, QP.ENTITY_MAP, QP.LOCAL, QP.VIA] ) @RequestParam queryParams: MultiValueMap<String, String> ): ResponseEntity<*> LongMethod:EntityHandler.kt$EntityHandler$@GetMapping(produces = [APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE, GEO_JSON_CONTENT_TYPE]) suspend fun getEntities( @RequestHeader httpHeaders: HttpHeaders, @AllowedParameters( implemented = [ QP.OPTIONS, QP.COUNT, QP.OFFSET, QP.LIMIT, QP.ID, QP.TYPE, QP.ID_PATTERN, QP.ATTRS, QP.Q, QP.GEOMETRY, QP.GEOREL, QP.COORDINATES, QP.GEOPROPERTY, QP.GEOMETRY_PROPERTY, QP.LANG, QP.SCOPEQ, QP.CONTAINED_BY, QP.JOIN, QP.JOIN_LEVEL, QP.DATASET_ID, ], notImplemented = [QP.FORMAT, QP.PICK, QP.OMIT, QP.EXPAND_VALUES, QP.CSF, QP.ENTITY_MAP, QP.LOCAL, QP.VIA] ) @RequestParam queryParams: MultiValueMap<String, String> ): ResponseEntity<*> + LongMethod:EntityEventService.kt$EntityEventService$private fun publishAttributeChangeEvent( updatedDetails: UpdatedDetails, sub: String?, tenantName: String, entityId: URI, entityTypesAndPayload: Pair<List<ExpandedTerm>, String>, serializedAttribute: Pair<ExpandedTerm, String>, overwrite: Boolean ) LongMethod:LinkedEntityServiceTests.kt$LinkedEntityServiceTests$@Test fun `it should inline entities up to the asked 2nd level`() LongMethod:PatchAttributeTests.kt$PatchAttributeTests.Companion$@JvmStatic fun mergePatchProvider(): Stream<Arguments> LongMethod:PatchAttributeTests.kt$PatchAttributeTests.Companion$@JvmStatic fun partialUpdatePatchProvider(): Stream<Arguments> diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/listener/IAMListener.kt b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/listener/IAMListener.kt index d4fa226e6..a7e32bec2 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/listener/IAMListener.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/listener/IAMListener.kt @@ -125,7 +125,7 @@ class IAMListener( // (if it no longer exists, it fails because of access rights checks) if (searchProperties.onOwnerDeleteCascadeEntities && subjectType == SubjectType.USER) { entityAccessRightsService.getEntitiesIdsOwnedBySubject(sub).getOrNull()?.forEach { entityId -> - entityService.deleteEntity(entityId, sub) + entityService.permanentlyDeleteEntity(entityId, sub) } Unit.right() } else Unit.right() diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/model/EntityAccessRights.kt b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/model/EntityAccessRights.kt index 022110962..7528f783c 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/model/EntityAccessRights.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/model/EntityAccessRights.kt @@ -6,6 +6,7 @@ import com.egm.stellio.shared.model.addNonReifiedProperty import com.egm.stellio.shared.model.addSubAttribute import com.egm.stellio.shared.util.AccessRight import com.egm.stellio.shared.util.AuthContextModel +import com.egm.stellio.shared.util.AuthContextModel.AUTH_PROP_IS_DELETED import com.egm.stellio.shared.util.AuthContextModel.AUTH_PROP_RIGHT import com.egm.stellio.shared.util.AuthContextModel.AUTH_PROP_SAP import com.egm.stellio.shared.util.AuthContextModel.AUTH_PROP_SUBJECT_INFO @@ -27,6 +28,7 @@ import java.net.URI data class EntityAccessRights( val id: URI, val types: List, + val isDeleted: Boolean = false, // right the current user has on the entity val right: AccessRight, val specificAccessPolicy: AuthContextModel.SpecificAccessPolicy? = null, @@ -55,6 +57,8 @@ data class EntityAccessRights( resultEntity[JSONLD_ID] = id.toString() resultEntity[JSONLD_TYPE] = types + if (isDeleted) + resultEntity[AUTH_PROP_IS_DELETED] = buildExpandedPropertyValue(true) resultEntity[AUTH_PROP_RIGHT] = buildExpandedPropertyValue(right.attributeName) specificAccessPolicy?.run { diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/AuthorizationService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/AuthorizationService.kt index c0dd9d0fe..c79064cfa 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/AuthorizationService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/AuthorizationService.kt @@ -23,6 +23,7 @@ interface AuthorizationService { suspend fun getAuthorizedEntities( entitiesQuery: EntitiesQueryFromGet, + includeDeleted: Boolean, contexts: List, sub: Option ): Either>> diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/DisabledAuthorizationService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/DisabledAuthorizationService.kt index f27debb04..5b56b8a42 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/DisabledAuthorizationService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/DisabledAuthorizationService.kt @@ -41,6 +41,7 @@ class DisabledAuthorizationService : AuthorizationService { override suspend fun getAuthorizedEntities( entitiesQuery: EntitiesQueryFromGet, + includeDeleted: Boolean, contexts: List, sub: Option ): Either>> = Pair(-1, emptyList()).right() diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/EnabledAuthorizationService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/EnabledAuthorizationService.kt index 53e3e31cf..65a64a3cf 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/EnabledAuthorizationService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/EnabledAuthorizationService.kt @@ -108,6 +108,7 @@ class EnabledAuthorizationService( override suspend fun getAuthorizedEntities( entitiesQuery: EntitiesQueryFromGet, + includeDeleted: Boolean, contexts: List, sub: Option ): Either>> = either { @@ -115,9 +116,8 @@ class EnabledAuthorizationService( val entitiesAccessRights = entityAccessRightsService.getSubjectAccessRights( sub, accessRights, - entitiesQuery.typeSelection, - entitiesQuery.ids, - entitiesQuery.paginationQuery + entitiesQuery, + includeDeleted ).bind() // for each entity user is admin or creator of, retrieve the full details of rights other users have on it @@ -148,7 +148,8 @@ class EnabledAuthorizationService( sub, accessRights, entitiesQuery.typeSelection, - entitiesQuery.ids + entitiesQuery.ids, + includeDeleted ).bind() Pair(count, entitiesAccessControlWithSubjectRights) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/EntityAccessRightsService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/EntityAccessRightsService.kt index e85454db8..fb6b3ce12 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/EntityAccessRightsService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/EntityAccessRightsService.kt @@ -18,13 +18,13 @@ import com.egm.stellio.search.common.util.toJsonString import com.egm.stellio.search.common.util.toList import com.egm.stellio.search.common.util.toOptionalEnum import com.egm.stellio.search.common.util.toUri +import com.egm.stellio.search.entity.model.EntitiesQueryFromGet import com.egm.stellio.shared.config.ApplicationProperties import com.egm.stellio.shared.model.APIException import com.egm.stellio.shared.model.AccessDeniedException import com.egm.stellio.shared.model.EntityTypeSelection import com.egm.stellio.shared.model.NgsiLdAttribute import com.egm.stellio.shared.model.ResourceNotFoundException -import com.egm.stellio.shared.queryparameter.PaginationQuery import com.egm.stellio.shared.util.AccessRight import com.egm.stellio.shared.util.AccessRight.CAN_ADMIN import com.egm.stellio.shared.util.AccessRight.CAN_READ @@ -220,30 +220,32 @@ class EntityAccessRightsService( suspend fun getSubjectAccessRights( sub: Option, accessRights: List, - type: EntityTypeSelection? = null, - ids: Set? = null, - paginationQuery: PaginationQuery, + entitiesQuery: EntitiesQueryFromGet, + includeDeleted: Boolean = false ): Either> = either { + val ids = entitiesQuery.ids + val typeSelection = entitiesQuery.typeSelection val subjectUuids = subjectReferentialService.getSubjectAndGroupsUUID(sub).bind() val isStellioAdmin = subjectReferentialService.hasStellioAdminRole(subjectUuids).bind() databaseClient .sql( """ - SELECT ep.entity_id, ep.types, ear.access_right, ep.specific_access_policy + SELECT ep.entity_id, ep.types, ear.access_right, ep.specific_access_policy, ep.deleted_at FROM entity_access_rights ear LEFT JOIN entity_payload ep ON ear.entity_id = ep.entity_id WHERE ${if (isStellioAdmin) "1 = 1" else "subject_id IN (:subject_uuids)" } ${if (accessRights.isNotEmpty()) " AND access_right IN (:access_rights)" else ""} - ${if (!type.isNullOrEmpty()) " AND (${buildTypeQuery(type)})" else ""} - ${if (!ids.isNullOrEmpty()) " AND ear.entity_id IN (:entities_ids)" else ""} + ${if (!typeSelection.isNullOrEmpty()) " AND (${buildTypeQuery(typeSelection)})" else ""} + ${if (ids.isNotEmpty()) " AND ear.entity_id IN (:entities_ids)" else ""} + ${if (!includeDeleted) " AND deleted_at IS NULL" else ""} ORDER BY entity_id LIMIT :limit OFFSET :offset; """.trimIndent() ) - .bind("limit", paginationQuery.limit) - .bind("offset", paginationQuery.offset) + .bind("limit", entitiesQuery.paginationQuery.limit) + .bind("offset", entitiesQuery.paginationQuery.offset) .let { if (!isStellioAdmin) it.bind("subject_uuids", subjectUuids) @@ -255,7 +257,7 @@ class EntityAccessRightsService( else it } .let { - if (!ids.isNullOrEmpty()) + if (ids.isNotEmpty()) it.bind("entities_ids", ids) else it } @@ -268,6 +270,7 @@ class EntityAccessRightsService( EntityAccessRights( ear.id, ear.types, + ear.isDeleted, entityAccessRights.maxOf { it.right }, ear.specificAccessPolicy ) @@ -278,7 +281,8 @@ class EntityAccessRightsService( sub: Option, accessRights: List, type: EntityTypeSelection? = null, - ids: Set? = null + ids: Set? = null, + includeDeleted: Boolean = false ): Either = either { val subjectUuids = subjectReferentialService.getSubjectAndGroupsUUID(sub).bind() val isStellioAdmin = subjectReferentialService.hasStellioAdminRole(subjectUuids).bind() @@ -293,6 +297,7 @@ class EntityAccessRightsService( ${if (accessRights.isNotEmpty()) " AND access_right IN (:access_rights)" else ""} ${if (!type.isNullOrEmpty()) " AND (${buildTypeQuery(type)})" else ""} ${if (!ids.isNullOrEmpty()) " AND ear.entity_id IN (:entities_ids)" else ""} + ${if (!includeDeleted) " AND deleted_at IS NULL" else ""} """.trimIndent() ) .let { @@ -443,6 +448,7 @@ class EntityAccessRightsService( return EntityAccessRights( id = toUri(row["entity_id"]), types = toList(row["types"]), + isDeleted = row["deleted_at"] != null, right = accessRight, specificAccessPolicy = toOptionalEnum(row["specific_access_policy"]) ) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/web/EntityAccessControlHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/web/EntityAccessControlHandler.kt index 996dd3985..7de1a6867 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/web/EntityAccessControlHandler.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/web/EntityAccessControlHandler.kt @@ -19,6 +19,7 @@ import com.egm.stellio.shared.model.toNgsiLdAttribute import com.egm.stellio.shared.model.toNgsiLdAttributes import com.egm.stellio.shared.queryparameter.AllowedParameters import com.egm.stellio.shared.queryparameter.QP +import com.egm.stellio.shared.queryparameter.QueryParameter import com.egm.stellio.shared.util.AccessRight import com.egm.stellio.shared.util.AuthContextModel.ALL_ASSIGNABLE_IAM_RIGHTS import com.egm.stellio.shared.util.AuthContextModel.ALL_IAM_RIGHTS @@ -70,10 +71,11 @@ class EntityAccessControlHandler( @GetMapping("/entities", produces = [MediaType.APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE]) suspend fun getAuthorizedEntities( @RequestHeader httpHeaders: HttpHeaders, - @AllowedParameters(implemented = [QP.ID, QP.TYPE, QP.ATTRS, QP.COUNT, QP.OFFSET, QP.LIMIT]) + @AllowedParameters(implemented = [QP.ID, QP.TYPE, QP.ATTRS, QP.COUNT, QP.OFFSET, QP.LIMIT, QP.INCLUDE_DELETED]) @RequestParam queryParams: MultiValueMap ): ResponseEntity<*> = either { val sub = getSubFromSecurityContext() + val includeDeleted = queryParams.getFirst(QueryParameter.INCLUDE_DELETED.key)?.toBoolean() == true val contexts = getAuthzContextFromLinkHeaderOrDefault(httpHeaders, applicationProperties.contexts).bind() val mediaType = getApplicableMediaType(httpHeaders).bind() @@ -91,6 +93,7 @@ class EntityAccessControlHandler( val (count, entities) = authorizationService.getAuthorizedEntities( entitiesQuery, + includeDeleted, contexts, sub ).bind() diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/discovery/service/AttributeService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/discovery/service/AttributeService.kt index eb9aecf56..c2d1720b6 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/discovery/service/AttributeService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/discovery/service/AttributeService.kt @@ -30,6 +30,7 @@ class AttributeService( """ SELECT DISTINCT(attribute_name) FROM temporal_entity_attribute + WHERE deleted_at IS NULL ORDER BY attribute_name """.trimIndent() ).allToMappedList { rowToAttributeNames(it) } @@ -42,7 +43,10 @@ class AttributeService( """ SELECT types, attribute_name FROM entity_payload - JOIN temporal_entity_attribute ON entity_payload.entity_id = temporal_entity_attribute.entity_id + JOIN temporal_entity_attribute + ON entity_payload.entity_id = temporal_entity_attribute.entity_id + AND temporal_entity_attribute.deleted_at IS NULL + WHERE entity_payload.deleted_at IS NULL ORDER BY attribute_name """.trimIndent() ).allToMappedList { rowToAttributeDetails(it) }.flatten().groupBy({ it.second }, { it.first }).toList() @@ -65,11 +69,14 @@ class AttributeService( WITH entities AS ( SELECT entity_id, attribute_name, attribute_type FROM temporal_entity_attribute - WHERE attribute_name = :attribute_name + WHERE attribute_name = :attribute_name + AND deleted_at IS NULL ) SELECT attribute_name, attribute_type, types, count(distinct(attribute_name)) as attribute_count FROM entity_payload - JOIN entities ON entity_payload.entity_id = entities.entity_id + JOIN entities + ON entity_payload.entity_id = entities.entity_id + AND entity_payload.deleted_at IS NULL GROUP BY types, attribute_name, attribute_type """.trimIndent() ) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/discovery/service/EntityTypeService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/discovery/service/EntityTypeService.kt index 07626ecb7..7a4d796c6 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/discovery/service/EntityTypeService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/discovery/service/EntityTypeService.kt @@ -29,6 +29,7 @@ class EntityTypeService( """ SELECT DISTINCT(unnest(types)) as type FROM entity_payload + WHERE deleted_at IS NULL ORDER BY type """.trimIndent() ).allToMappedList { rowToType(it) } @@ -41,7 +42,10 @@ class EntityTypeService( """ SELECT unnest(types) as type, attribute_name FROM entity_payload - JOIN temporal_entity_attribute ON entity_payload.entity_id = temporal_entity_attribute.entity_id + JOIN temporal_entity_attribute + ON entity_payload.entity_id = temporal_entity_attribute.entity_id + AND temporal_entity_attribute.deleted_at IS NULL + WHERE entity_payload.deleted_at IS NULL ORDER BY type """.trimIndent() ).allToMappedList { rowToEntityType(it) }.groupBy({ it.first }, { it.second }).toList() @@ -65,10 +69,12 @@ class EntityTypeService( SELECT entity_id FROM entity_payload WHERE :type_name = any (types) + AND deleted_at IS NULL ) SELECT attribute_name, attribute_type, (select count(entity_id) from entities) as entity_count FROM temporal_entity_attribute WHERE entity_id IN (SELECT entity_id FROM entities) + AND deleted_at IS NULL GROUP BY attribute_name, attribute_type """.trimIndent() ) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/model/Attribute.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/model/Attribute.kt index bca913e99..f3d4474af 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/model/Attribute.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/model/Attribute.kt @@ -1,18 +1,33 @@ package com.egm.stellio.search.entity.model import com.egm.stellio.shared.model.ExpandedTerm +import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_LANGUAGE +import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_LANGUAGEMAP_TERM +import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_OBJECT +import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE_TERM +import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_VALUE +import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_VALUE_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_GEOPROPERTY_TYPE +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_GEOPROPERTY_VALUE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_GEOPROPERTY_VALUES import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_JSONPROPERTY_TYPE +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_JSONPROPERTY_VALUE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_JSONPROPERTY_VALUES import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_LANGUAGEPROPERTY_TYPE +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_LANGUAGEPROPERTY_VALUE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_LANGUAGEPROPERTY_VALUES +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_NONE_TERM +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_NULL import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_PROPERTY_TYPE +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_PROPERTY_VALUE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_PROPERTY_VALUES +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_RELATIONSHIP_OBJECT import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_RELATIONSHIP_OBJECTS import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_RELATIONSHIP_TYPE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_VOCABPROPERTY_TYPE +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_VOCABPROPERTY_VALUE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_VOCABPROPERTY_VALUES +import com.egm.stellio.shared.util.JsonUtils.serializeObject import io.r2dbc.postgresql.codec.Json import org.springframework.data.annotation.Id import java.net.URI @@ -29,6 +44,7 @@ data class Attribute( val datasetId: URI? = null, val createdAt: ZonedDateTime, val modifiedAt: ZonedDateTime? = null, + val deletedAt: ZonedDateTime? = null, val payload: Json ) { enum class AttributeValueType { @@ -63,6 +79,21 @@ data class Attribute( VocabProperty -> NGSILD_VOCABPROPERTY_TYPE.uri } + /** + * Returns the expanded name of the member who holds the value of the attribute. + * + * For instance, https://uri.etsi.org/ngsi-ld/hasJSON if it is a JsonProperty + */ + fun toExpandedValueMember(): String = + when (this) { + Property -> NGSILD_PROPERTY_VALUE + Relationship -> NGSILD_RELATIONSHIP_OBJECT + GeoProperty -> NGSILD_GEOPROPERTY_VALUE + JsonProperty -> NGSILD_JSONPROPERTY_VALUE + LanguageProperty -> NGSILD_LANGUAGEPROPERTY_VALUE + VocabProperty -> NGSILD_VOCABPROPERTY_VALUE + } + /** * Returns the key of the member for the simplified representation of the attribute, as defined in 4.5.9 */ @@ -75,5 +106,31 @@ data class Attribute( LanguageProperty -> NGSILD_LANGUAGEPROPERTY_VALUES VocabProperty -> NGSILD_VOCABPROPERTY_VALUES } + + fun toNullCompactedRepresentation(): Map = + when (this) { + Property, GeoProperty, JsonProperty, VocabProperty -> + mapOf( + JSONLD_TYPE_TERM to this.name, + JSONLD_VALUE_TERM to NGSILD_NULL + ) + Relationship -> + mapOf( + JSONLD_TYPE_TERM to this.name, + JSONLD_OBJECT to NGSILD_NULL + ) + LanguageProperty -> + mapOf( + JSONLD_TYPE_TERM to this.name, + JSONLD_LANGUAGEMAP_TERM to mapOf(NGSILD_NONE_TERM to NGSILD_NULL) + ) + } + + fun toNullValue(): String = + when (this) { + Property, GeoProperty, JsonProperty, VocabProperty, Relationship -> NGSILD_NULL + LanguageProperty -> + serializeObject(listOf(mapOf(JSONLD_VALUE to NGSILD_NULL, JSONLD_LANGUAGE to NGSILD_NONE_TERM))) + } } } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/model/Entity.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/model/Entity.kt index e81816479..f54601471 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/model/Entity.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/model/Entity.kt @@ -1,5 +1,6 @@ package com.egm.stellio.search.entity.model +import com.egm.stellio.shared.model.ExpandedEntity import com.egm.stellio.shared.model.ExpandedTerm import com.egm.stellio.shared.util.AuthContextModel import com.egm.stellio.shared.util.AuthContextModel.SpecificAccessPolicy @@ -7,6 +8,7 @@ import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_ID import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_VALUE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_CREATED_AT_PROPERTY +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DELETED_AT_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_MODIFIED_AT_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_SCOPE_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.buildExpandedPropertyValue @@ -44,4 +46,18 @@ data class Entity( return resultEntity } + + companion object { + + fun toExpandedDeletedEntity( + entityId: URI, + deletedAt: ZonedDateTime + ): ExpandedEntity = + ExpandedEntity( + members = mapOf( + JSONLD_ID to entityId, + NGSILD_DELETED_AT_PROPERTY to buildNonReifiedTemporalValue(deletedAt) + ) + ) + } } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/model/UpdateResult.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/model/UpdateResult.kt index 9ccf6a575..b854ede7d 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/model/UpdateResult.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/model/UpdateResult.kt @@ -53,6 +53,7 @@ data class UpdateAttributeResult( UpdateOperationResult.APPENDED, UpdateOperationResult.REPLACED, UpdateOperationResult.UPDATED, + UpdateOperationResult.DELETED, UpdateOperationResult.IGNORED ) } @@ -61,10 +62,11 @@ enum class UpdateOperationResult { APPENDED, REPLACED, UPDATED, + DELETED, IGNORED, FAILED; - fun isSuccessResult(): Boolean = listOf(APPENDED, REPLACED, UPDATED).contains(this) + fun isSuccessResult(): Boolean = listOf(APPENDED, REPLACED, UPDATED, DELETED).contains(this) } fun updateResultFromDetailedResult(updateStatuses: List): UpdateResult { diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityAttributeService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityAttributeService.kt index adf1ff565..72dc47c7b 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityAttributeService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityAttributeService.kt @@ -28,6 +28,7 @@ import com.egm.stellio.search.entity.model.UpdateOperationResult import com.egm.stellio.search.entity.model.UpdateResult import com.egm.stellio.search.entity.model.updateResultFromDetailedResult import com.egm.stellio.search.entity.util.guessAttributeValueType +import com.egm.stellio.search.entity.util.hasNgsiLdNullValue import com.egm.stellio.search.entity.util.mergePatch import com.egm.stellio.search.entity.util.partialUpdatePatch import com.egm.stellio.search.entity.util.prepareAttributes @@ -35,6 +36,7 @@ import com.egm.stellio.search.entity.util.toAttributeMetadata import com.egm.stellio.search.entity.util.toExpandedAttributeInstance import com.egm.stellio.search.temporal.model.AttributeInstance import com.egm.stellio.search.temporal.service.AttributeInstanceService +import com.egm.stellio.shared.config.ApplicationProperties import com.egm.stellio.shared.model.APIException import com.egm.stellio.shared.model.BadRequestDataException import com.egm.stellio.shared.model.ExpandedAttribute @@ -56,6 +58,7 @@ import com.egm.stellio.shared.model.isAttributeOfType import com.egm.stellio.shared.model.toNgsiLdEntity import com.egm.stellio.shared.util.AttributeType import com.egm.stellio.shared.util.AuthContextModel +import com.egm.stellio.shared.util.JsonLdUtils import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_JSONPROPERTY_VALUE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_LANGUAGEPROPERTY_VALUE @@ -70,6 +73,7 @@ import com.egm.stellio.shared.util.JsonUtils.serializeObject import com.egm.stellio.shared.util.Sub import com.egm.stellio.shared.util.attributeNotFoundMessage import com.egm.stellio.shared.util.entityNotFoundMessage +import com.egm.stellio.shared.util.ngsiLdDateTime import io.r2dbc.postgresql.codec.Json import org.slf4j.LoggerFactory import org.springframework.r2dbc.core.DatabaseClient @@ -77,14 +81,15 @@ import org.springframework.r2dbc.core.bind import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional import java.net.URI -import java.time.ZoneOffset import java.time.ZonedDateTime -import java.util.UUID +import java.util.* +import kotlin.collections.map @Service class EntityAttributeService( private val databaseClient: DatabaseClient, - private val attributeInstanceService: AttributeInstanceService + private val attributeInstanceService: AttributeInstanceService, + private val applicationProperties: ApplicationProperties ) { private val logger = LoggerFactory.getLogger(javaClass) @@ -99,6 +104,12 @@ class EntityAttributeService( VALUES (:id, :entity_id, :attribute_name, :attribute_type, :attribute_value_type, :created_at, :dataset_id, :payload) + ON CONFLICT (entity_id, attribute_name, dataset_id) + DO UPDATE SET deleted_at = null, + attribute_type = :attribute_type, + attribute_value_type = :attribute_value_type, + modified_at = :created_at, + payload = :payload """.trimIndent() ) .bind("id", attribute.id) @@ -111,31 +122,6 @@ class EntityAttributeService( .bind("payload", attribute.payload) .execute() - @Transactional - suspend fun updateOnReplace( - attributeUUID: UUID, - attributeMetadata: AttributeMetadata, - modifiedAt: ZonedDateTime, - payload: String - ): Either = - databaseClient.sql( - """ - UPDATE temporal_entity_attribute - SET - attribute_type = :attribute_type, - attribute_value_type = :attribute_value_type, - modified_at = :modified_at, - payload = :payload - WHERE id = :id - """.trimIndent() - ) - .bind("id", attributeUUID) - .bind("attribute_type", attributeMetadata.type.toString()) - .bind("attribute_value_type", attributeMetadata.valueType.toString()) - .bind("modified_at", modifiedAt) - .bind("payload", Json.of(payload)) - .execute() - @Transactional suspend fun updateOnUpdate( attributeUUID: UUID, @@ -169,7 +155,7 @@ class EntityAttributeService( contexts: List, sub: String? = null ): Either = either { - val createdAt = ZonedDateTime.now(ZoneOffset.UTC) + val createdAt = ngsiLdDateTime() val expandedEntity = expandJsonLdEntity(payload, contexts) val ngsiLdEntity = expandedEntity.toNgsiLdEntity().bind() ngsiLdEntity.prepareAttributes() @@ -268,32 +254,15 @@ class EntityAttributeService( attributeMetadata.datasetId, attribute.entityId ) - updateOnReplace( - attribute.id, + deleteAttribute(attribute.entityId, attribute.attributeName, attribute.datasetId, false, createdAt).bind() + addAttribute( + attribute.entityId, + attribute.attributeName, attributeMetadata, createdAt, - serializeObject(attributePayload) + attributePayload, + sub ).bind() - - val attributeInstance = AttributeInstance( - attributeUuid = attribute.id, - timeProperty = AttributeInstance.TemporalProperty.MODIFIED_AT, - time = createdAt, - attributeMetadata = attributeMetadata, - payload = attributePayload, - sub = sub - ) - attributeInstanceService.create(attributeInstance).bind() - - if (attributeMetadata.observedAt != null) { - val attributeObservedAtInstance = AttributeInstance( - attributeUuid = attribute.id, - time = attributeMetadata.observedAt, - attributeMetadata = attributeMetadata, - payload = attributePayload - ) - attributeInstanceService.create(attributeObservedAtInstance).bind() - } } @Transactional @@ -329,79 +298,123 @@ class EntityAttributeService( } @Transactional - suspend fun deleteAttributes(entityId: URI): Either { - val uuids = databaseClient.sql( + suspend fun deleteAttributes(entityId: URI, deletedAt: ZonedDateTime): Either = either { + val attributesToDelete = getForEntity(entityId, emptySet(), emptySet()) + deleteSelectedAttributes(attributesToDelete, deletedAt).bind() + } + + @Transactional + suspend fun deleteAttribute( + entityId: URI, + attributeName: String, + datasetId: URI?, + deleteAll: Boolean = false, + deletedAt: ZonedDateTime + ): Either = either { + logger.debug("Deleting attribute {} from entity {} (all: {})", attributeName, entityId, deleteAll) + val attributesToDelete = + if (deleteAll) + getForEntity(entityId, setOf(attributeName), emptySet()) + else + listOf(getForEntityAndAttribute(entityId, attributeName, datasetId).bind()) + deleteSelectedAttributes(attributesToDelete, deletedAt).bind() + } + + @Transactional + internal suspend fun deleteSelectedAttributes( + attributesToDelete: List, + deletedAt: ZonedDateTime + ): Either = either { + if (attributesToDelete.isEmpty()) return Unit.right() + val attributesToDeleteWithPayload = attributesToDelete.map { + Triple( + it, + deletedAt, + JsonLdUtils.expandAttribute( + it.attributeName, + it.attributeType.toNullCompactedRepresentation(), + listOf(applicationProperties.contexts.core) + ).second[0] + ) + } + + databaseClient.sql( """ - DELETE FROM temporal_entity_attribute - WHERE entity_id = :entity_id - RETURNING id + UPDATE temporal_entity_attribute + SET deleted_at = new.deleted_at, + payload = new.payload + FROM (VALUES :values) AS new(uuid, deleted_at, payload) + WHERE temporal_entity_attribute.id = new.uuid """.trimIndent() ) - .bind("entity_id", entityId) + .bind("values", attributesToDeleteWithPayload.map { arrayOf(it.first.id, it.second, it.third.toJson()) }) .allToMappedList { - toUuid(it["id"]) + Triple( + toUuid(it["id"]), + Attribute.AttributeType.valueOf(it["attribute_type"] as String), + it["attribute_name"] as String + ) } - return if (uuids.isNotEmpty()) - attributeInstanceService.deleteInstancesOfEntity(uuids) - else Unit.right() + attributesToDeleteWithPayload.forEach { (attribute, deletedAt, expandedAttributePayload) -> + attributeInstanceService.addDeletedAttributeInstance( + attributeUuid = attribute.id, + value = attribute.attributeType.toNullValue(), + deletedAt = deletedAt, + attributeValues = expandedAttributePayload + ).bind() + } } @Transactional - suspend fun deleteAttribute( + suspend fun permanentlyDeleteAttribute( entityId: URI, attributeName: String, datasetId: URI?, deleteAll: Boolean = false - ): Either = - either { - logger.debug("Deleting attribute {} from entity {} (all: {})", attributeName, entityId, deleteAll) - if (deleteAll) { - attributeInstanceService.deleteAllInstancesOfAttribute(entityId, attributeName).bind() - deleteAllInstances(entityId, attributeName).bind() - } else { - attributeInstanceService.deleteInstancesOfAttribute(entityId, attributeName, datasetId).bind() - deleteSpecificInstance(entityId, attributeName, datasetId).bind() - } - } + ): Either = either { + logger.debug("Permanently deleting attribute {} from entity {} (all: {})", attributeName, entityId, deleteAll) + val attributesToDelete = + if (deleteAll) + getForEntity(entityId, setOf(attributeName), emptySet(), false) + else + listOf(getForEntityAndAttribute(entityId, attributeName, datasetId).bind()) - @Transactional - suspend fun deleteSpecificInstance( - entityId: URI, - attributeName: String, - datasetId: URI? - ): Either = databaseClient.sql( """ DELETE FROM temporal_entity_attribute - WHERE entity_id = :entity_id - ${datasetId.toDatasetIdFilter()} - AND attribute_name = :attribute_name + WHERE id IN(:uuids) """.trimIndent() ) - .bind("entity_id", entityId) - .bind("attribute_name", attributeName) - .let { - if (datasetId != null) it.bind("dataset_id", datasetId) - else it - } + .bind("uuids", attributesToDelete.map { it.id }) .execute() + if (deleteAll) + attributeInstanceService.deleteAllInstancesOfAttribute(entityId, attributeName).bind() + else + attributeInstanceService.deleteInstancesOfAttribute(entityId, attributeName, datasetId).bind() + } + @Transactional - suspend fun deleteAllInstances( + suspend fun permanentlyDeleteAttributes( entityId: URI, - attributeName: String - ): Either = - databaseClient.sql( + ): Either = either { + logger.debug("Permanently deleting all attributes from entity {}", entityId) + + val deletedTeas = databaseClient.sql( """ DELETE FROM temporal_entity_attribute WHERE entity_id = :entity_id - AND attribute_name = :attribute_name + RETURNING id """.trimIndent() ) .bind("entity_id", entityId) - .bind("attribute_name", attributeName) - .execute() + .allToMappedList { toUuid(it["id"]) } + + if (deletedTeas.isNotEmpty()) + attributeInstanceService.deleteInstancesOfEntity(deletedTeas).bind() + else Unit.right() + } suspend fun getForEntities( entitiesIds: List, @@ -426,7 +439,7 @@ class EntityAttributeService( val selectQuery = """ SELECT id, entity_id, attribute_name, attribute_type, attribute_value_type, created_at, modified_at, - dataset_id, payload + deleted_at, dataset_id, payload FROM temporal_entity_attribute WHERE entity_id IN (:entities_ids) $filterOnAttributes @@ -440,7 +453,12 @@ class EntityAttributeService( .allToMappedList { rowToAttribute(it) } } - suspend fun getForEntity(id: URI, attrs: Set, datasetIds: Set): List { + suspend fun getForEntity( + id: URI, + attrs: Set, + datasetIds: Set, + excludedDeleted: Boolean = true + ): List { val filterOnAttributes = if (attrs.isNotEmpty()) " AND " + attrs.joinToString( @@ -463,6 +481,7 @@ class EntityAttributeService( dataset_id, payload FROM temporal_entity_attribute WHERE entity_id = :entity_id + ${if (excludedDeleted) " and deleted_at is null " else ""} $filterOnAttributes $filterOnDatasetId """.trimIndent() @@ -500,33 +519,6 @@ class EntityAttributeService( } } - suspend fun hasAttribute( - id: URI, - attributeName: String, - datasetId: URI? = null - ): Either { - val selectQuery = - """ - SELECT count(entity_id) as count - FROM temporal_entity_attribute - WHERE entity_id = :entity_id - ${datasetId.toDatasetIdFilter()} - AND attribute_name = :attribute_name - """.trimIndent() - - return databaseClient - .sql(selectQuery) - .bind("entity_id", id) - .bind("attribute_name", attributeName) - .let { - if (datasetId != null) it.bind("dataset_id", datasetId) - else it - } - .oneToResult { - it["count"] as Long == 1L - } - } - private fun rowToAttribute(row: Map) = Attribute( id = toUuid(row["id"]), @@ -539,6 +531,7 @@ class EntityAttributeService( datasetId = toOptionalUri(row["dataset_id"]), createdAt = toZonedDateTime(row["created_at"]), modifiedAt = toOptionalZonedDateTime(row["modified_at"]), + deletedAt = toOptionalZonedDateTime(row["deleted_at"]), payload = toJson(row["payload"]) ) @@ -564,6 +557,7 @@ class EntityAttributeService( from temporal_entity_attribute where entity_id = :entity_id and attribute_name = :attribute_name + and deleted_at is null $datasetIdFilter ) as attributeNameExists; """.trimIndent() @@ -618,8 +612,7 @@ class EntityAttributeService( UpdateAttributeResult( ngsiLdAttribute.name, ngsiLdAttributeInstance.datasetId, - UpdateOperationResult.APPENDED, - null + UpdateOperationResult.APPENDED ) }.bind() } else if (disallowOverwrite) { @@ -643,8 +636,7 @@ class EntityAttributeService( UpdateAttributeResult( ngsiLdAttribute.name, ngsiLdAttributeInstance.datasetId, - UpdateOperationResult.REPLACED, - null + UpdateOperationResult.REPLACED ) }.bind() } @@ -670,10 +662,11 @@ class EntityAttributeService( ngsiLdAttribute.name, ngsiLdAttributeInstance.datasetId )!! - if (currentAttribute != null) { - replaceAttribute( - currentAttribute, - ngsiLdAttribute, + + if (currentAttribute == null) { + addAttribute( + entityUri, + ngsiLdAttribute.name, attributeMetadata, createdAt, attributePayload, @@ -682,24 +675,34 @@ class EntityAttributeService( UpdateAttributeResult( ngsiLdAttribute.name, ngsiLdAttributeInstance.datasetId, - UpdateOperationResult.REPLACED, - null + UpdateOperationResult.APPENDED ) }.bind() - } else { - addAttribute( + } else if (hasNgsiLdNullValue(attributePayload, currentAttribute.attributeType)) { + deleteAttribute( entityUri, ngsiLdAttribute.name, - attributeMetadata, + ngsiLdAttributeInstance.datasetId, + false, + createdAt + ).map { + UpdateAttributeResult( + ngsiLdAttribute.name, + ngsiLdAttributeInstance.datasetId, + UpdateOperationResult.DELETED + ) + }.bind() + } else { + partialUpdateAttribute( + entityUri, + Pair(ngsiLdAttribute.name, listOf(attributePayload)), createdAt, - attributePayload, sub ).map { UpdateAttributeResult( ngsiLdAttribute.name, ngsiLdAttributeInstance.datasetId, - UpdateOperationResult.APPENDED, - null + UpdateOperationResult.REPLACED ) }.bind() } @@ -715,17 +718,33 @@ class EntityAttributeService( ): Either = either { val attributeName = expandedAttribute.first val attributeValues = expandedAttribute.second[0] - logger.debug( - "Updating attribute {} of entity {} with values: {}", - attributeName, - entityId, - attributeValues - ) + logger.debug("Partial updating attribute {} in entity {}", attributeName, entityId) val datasetId = attributeValues.getDatasetId() - val exists = hasAttribute(entityId, attributeName, datasetId).bind() + val currentAttribute = getForEntityAndAttribute(entityId, attributeName, datasetId).fold({ null }, { it }) val updateAttributeResult = - if (exists) { + if (currentAttribute == null) { + UpdateAttributeResult( + attributeName, + datasetId, + UpdateOperationResult.FAILED, + "Unknown attribute $attributeName with datasetId $datasetId in entity $entityId" + ) + } else if (hasNgsiLdNullValue(attributeValues, currentAttribute.attributeType)) { + deleteAttribute( + entityId, + attributeName, + datasetId, + false, + modifiedAt + ).map { + UpdateAttributeResult( + attributeName, + datasetId, + UpdateOperationResult.DELETED + ) + }.bind() + } else { // first update payload in temporal entity attribute val attribute = getForEntityAndAttribute(entityId, attributeName, datasetId).bind() attributeValues[JSONLD_TYPE]?.let { @@ -752,15 +771,7 @@ class EntityAttributeService( UpdateAttributeResult( attributeName, datasetId, - UpdateOperationResult.UPDATED, - null - ) - } else { - UpdateAttributeResult( - attributeName, - datasetId, - UpdateOperationResult.FAILED, - "Unknown attribute $attributeName with datasetId $datasetId in entity $entityId" + UpdateOperationResult.UPDATED ) } @@ -802,7 +813,7 @@ class EntityAttributeService( ).bind() } else { logger.debug("Adding instance to attribute {} to entity {}", currentAttribute.attributeName, entityUri) - attributeInstanceService.addAttributeInstance( + attributeInstanceService.addObservedAttributeInstance( currentAttribute.id, attributeMetadata, expandedAttributes[currentAttribute.attributeName]!!.first() @@ -831,7 +842,7 @@ class EntityAttributeService( ngsiLdAttributeInstance.datasetId )!! - if (currentAttribute == null) { + if (currentAttribute == null) addAttribute( entityUri, ngsiLdAttribute.name, @@ -843,11 +854,24 @@ class EntityAttributeService( UpdateAttributeResult( ngsiLdAttribute.name, ngsiLdAttributeInstance.datasetId, - UpdateOperationResult.APPENDED, - null + UpdateOperationResult.APPENDED ) }.bind() - } else { + else if (hasNgsiLdNullValue(attributePayload, currentAttribute.attributeType)) + deleteAttribute( + entityUri, + ngsiLdAttribute.name, + ngsiLdAttributeInstance.datasetId, + false, + createdAt + ).map { + UpdateAttributeResult( + ngsiLdAttribute.name, + ngsiLdAttributeInstance.datasetId, + UpdateOperationResult.DELETED + ) + }.bind() + else mergeAttribute( currentAttribute, ngsiLdAttribute.name, @@ -860,11 +884,9 @@ class EntityAttributeService( UpdateAttributeResult( ngsiLdAttribute.name, ngsiLdAttributeInstance.datasetId, - UpdateOperationResult.UPDATED, - null + UpdateOperationResult.UPDATED ) }.bind() - } } }.fold({ it.left() }, { updateResultFromDetailedResult(it).right() }) @@ -903,8 +925,7 @@ class EntityAttributeService( UpdateAttributeResult( ngsiLdAttribute.name, ngsiLdAttributeInstance.datasetId, - UpdateOperationResult.REPLACED, - null + UpdateOperationResult.REPLACED ) } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityEventService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityEventService.kt index 8586746ef..33d424f90 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityEventService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityEventService.kt @@ -188,6 +188,20 @@ class EntityEventService( ) ) + UpdateOperationResult.DELETED -> + publishEntityEvent( + AttributeDeleteEvent( + sub, + tenantName, + entityId, + entityTypesAndPayload.first, + serializedAttribute.first, + updatedDetails.datasetId, + serializedAttribute.second, + emptyList() + ) + ) + else -> logger.warn( "Received an unexpected result (${updatedDetails.updateOperationResult} " + diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityQueryService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityQueryService.kt index 7c756b569..d97eaadae 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityQueryService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityQueryService.kt @@ -18,14 +18,12 @@ import com.egm.stellio.search.entity.model.EntitiesQueryFromPost import com.egm.stellio.search.entity.model.Entity import com.egm.stellio.search.entity.util.rowToEntity import com.egm.stellio.shared.model.APIException -import com.egm.stellio.shared.model.AlreadyExistsException import com.egm.stellio.shared.model.ExpandedEntity import com.egm.stellio.shared.model.ResourceNotFoundException import com.egm.stellio.shared.util.Sub import com.egm.stellio.shared.util.buildQQuery import com.egm.stellio.shared.util.buildScopeQQuery import com.egm.stellio.shared.util.buildTypeQuery -import com.egm.stellio.shared.util.entityAlreadyExistsMessage import com.egm.stellio.shared.util.entityNotFoundMessage import org.springframework.r2dbc.core.DatabaseClient import org.springframework.stereotype.Service @@ -73,6 +71,13 @@ class EntityQueryService( suspend fun queryEntities( entitiesQuery: EntitiesQuery, accessRightFilter: () -> String? + ): List = + queryEntities(entitiesQuery, true, accessRightFilter) + + suspend fun queryEntities( + entitiesQuery: EntitiesQuery, + excludedDeleted: Boolean = true, + accessRightFilter: () -> String? ): List { val filterQuery = buildFullEntitiesFilter(entitiesQuery, accessRightFilter) @@ -81,8 +86,10 @@ class EntityQueryService( SELECT DISTINCT(entity_payload.entity_id) FROM entity_payload LEFT JOIN temporal_entity_attribute tea - ON tea.entity_id = entity_payload.entity_id + ON tea.entity_id = entity_payload.entity_id + ${if (excludedDeleted) " AND tea.deleted_at is null " else ""} WHERE $filterQuery + ${if (excludedDeleted) " AND entity_payload.deleted_at is null " else ""} ORDER BY entity_id LIMIT :limit OFFSET :offset @@ -98,6 +105,13 @@ class EntityQueryService( suspend fun queryEntitiesCount( entitiesQuery: EntitiesQuery, accessRightFilter: () -> String? + ): Either = + queryEntitiesCount(entitiesQuery, true, accessRightFilter) + + suspend fun queryEntitiesCount( + entitiesQuery: EntitiesQuery, + excludedDeleted: Boolean = true, + accessRightFilter: () -> String? ): Either { val filterQuery = buildFullEntitiesFilter(entitiesQuery, accessRightFilter) @@ -107,7 +121,9 @@ class EntityQueryService( FROM entity_payload LEFT JOIN temporal_entity_attribute tea ON tea.entity_id = entity_payload.entity_id + ${if (excludedDeleted) " AND tea.deleted_at is null " else ""} WHERE $filterQuery + ${if (excludedDeleted) " AND entity_payload.deleted_at is null " else ""} """.trimIndent() return databaseClient @@ -232,10 +248,7 @@ class EntityQueryService( .bind("entities_ids", entitiesIds) .allToMappedList { it.rowToEntity() } - suspend fun checkEntityExistence( - entityId: URI, - inverse: Boolean = false - ): Either { + suspend fun checkEntityExistence(entityId: URI, allowDeleted: Boolean = false): Either { val selectQuery = """ select @@ -243,6 +256,7 @@ class EntityQueryService( select 1 from entity_payload where entity_id = :entity_id + ${if (!allowDeleted) " and deleted_at is null " else ""} ) as entityExists; """.trimIndent() @@ -251,15 +265,32 @@ class EntityQueryService( .bind("entity_id", entityId) .oneToResult { it["entityExists"] as Boolean } .flatMap { - if (it && !inverse || !it && inverse) + if (it) Unit.right() - else if (it) - AlreadyExistsException(entityAlreadyExistsMessage(entityId.toString())).left() else ResourceNotFoundException(entityNotFoundMessage(entityId.toString())).left() } } + /** + * Used for checks before creating a (temporal) entity. Allows to know if the entity does not exist, + * or, if it exists, whether it is currently deleted (in which case, it may be possible to create it again + * if authorized) + */ + suspend fun isMarkedAsDeleted(entityId: URI): Either { + val selectQuery = + """ + select entity_id, deleted_at + from entity_payload + where entity_id = :entity_id + """.trimIndent() + + return databaseClient + .sql(selectQuery) + .bind("entity_id", entityId) + .oneToResult { it["deleted_at"] != null } + } + suspend fun filterExistingEntitiesAsIds(entitiesIds: List): List { if (entitiesIds.isEmpty()) { return emptyList() @@ -270,6 +301,7 @@ class EntityQueryService( select entity_id from entity_payload where entity_id in (:entities_ids) + and deleted_at is null """.trimIndent() return databaseClient diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityService.kt index 800d741a5..cac3a7d27 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityService.kt @@ -1,6 +1,9 @@ package com.egm.stellio.search.entity.service import arrow.core.Either +import arrow.core.Either.Left +import arrow.core.Either.Right +import arrow.core.left import arrow.core.raise.either import arrow.core.right import arrow.core.toOption @@ -24,7 +27,9 @@ import com.egm.stellio.search.entity.model.updateResultFromDetailedResult import com.egm.stellio.search.entity.util.prepareAttributes import com.egm.stellio.search.entity.util.rowToEntity import com.egm.stellio.search.scope.ScopeService +import com.egm.stellio.search.temporal.model.AttributeInstance.TemporalProperty import com.egm.stellio.shared.model.APIException +import com.egm.stellio.shared.model.AlreadyExistsException import com.egm.stellio.shared.model.ExpandedAttribute import com.egm.stellio.shared.model.ExpandedAttributeInstances import com.egm.stellio.shared.model.ExpandedAttributes @@ -35,10 +40,12 @@ import com.egm.stellio.shared.model.addSysAttrs import com.egm.stellio.shared.model.toExpandedAttributes import com.egm.stellio.shared.model.toNgsiLdAttributes import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_EXPANDED_ENTITY_SPECIFIC_MEMBERS +import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_ID import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_SCOPE_PROPERTY import com.egm.stellio.shared.util.JsonUtils.serializeObject import com.egm.stellio.shared.util.Sub +import com.egm.stellio.shared.util.entityAlreadyExistsMessage import com.egm.stellio.shared.util.getSpecificAccessPolicy import com.egm.stellio.shared.util.ngsiLdDateTime import io.r2dbc.postgresql.codec.Json @@ -48,7 +55,6 @@ import org.springframework.r2dbc.core.bind import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional import java.net.URI -import java.time.ZoneOffset import java.time.ZonedDateTime @Service @@ -68,14 +74,23 @@ class EntityService( expandedEntity: ExpandedEntity, sub: Sub? = null ): Either = either { - authorizationService.userCanCreateEntities(sub.toOption()).bind() - entityQueryService.checkEntityExistence(ngsiLdEntity.id, true).bind() + entityQueryService.isMarkedAsDeleted(ngsiLdEntity.id).let { + when (it) { + is Left -> authorizationService.userCanCreateEntities(sub.toOption()).bind() + is Right -> + if (!it.value) + AlreadyExistsException(entityAlreadyExistsMessage(ngsiLdEntity.id.toString())).left().bind() + else + authorizationService.userCanAdminEntity(ngsiLdEntity.id, sub.toOption()).bind() + } + } val createdAt = ngsiLdDateTime() val attributesMetadata = ngsiLdEntity.prepareAttributes().bind() logger.debug("Creating entity {}", ngsiLdEntity.id) - createEntityPayload(ngsiLdEntity, expandedEntity, createdAt, sub).bind() + createEntityPayload(ngsiLdEntity, expandedEntity, createdAt).bind() + scopeService.createHistory(ngsiLdEntity, createdAt, sub).bind() entityAttributeService.createAttributes( ngsiLdEntity, expandedEntity, @@ -96,14 +111,20 @@ class EntityService( suspend fun createEntityPayload( ngsiLdEntity: NgsiLdEntity, expandedEntity: ExpandedEntity, - createdAt: ZonedDateTime, - sub: Sub? = null + createdAt: ZonedDateTime ): Either = either { val specificAccessPolicy = ngsiLdEntity.getSpecificAccessPolicy()?.bind() databaseClient.sql( """ INSERT INTO entity_payload (entity_id, types, scopes, created_at, payload, specific_access_policy) VALUES (:entity_id, :types, :scopes, :created_at, :payload, :specific_access_policy) + ON CONFLICT (entity_id) + DO UPDATE SET types = :types, + scopes = :scopes, + modified_at = :created_at, + deleted_at = null, + payload = :payload, + specific_access_policy = :specific_access_policy """.trimIndent() ) .bind("entity_id", ngsiLdEntity.id) @@ -113,9 +134,6 @@ class EntityService( .bind("payload", Json.of(serializeObject(expandedEntity.populateCreationTimeDate(createdAt).members))) .bind("specific_access_policy", specificAccessPolicy?.toString()) .execute() - .map { - scopeService.createHistory(ngsiLdEntity, createdAt, sub) - } } @Transactional @@ -129,7 +147,10 @@ class EntityService( authorizationService.userCanUpdateEntity(entityId, sub.toOption()).bind() val (coreAttrs, otherAttrs) = - expandedAttributes.toList().partition { JSONLD_EXPANDED_ENTITY_SPECIFIC_MEMBERS.contains(it.first) } + expandedAttributes.toList() + // remove @id if it is present (optional as per 5.4) + .filter { it.first != JSONLD_ID } + .partition { JSONLD_EXPANDED_ENTITY_SPECIFIC_MEMBERS.contains(it.first) } val mergedAt = ngsiLdDateTime() logger.debug("Merging entity {}", entityId) @@ -173,13 +194,14 @@ class EntityService( entityQueryService.checkEntityExistence(entityId).bind() authorizationService.userCanUpdateEntity(entityId, sub.toOption()).bind() - val replacedAt = ngsiLdDateTime() val attributesMetadata = ngsiLdEntity.prepareAttributes().bind() logger.debug("Replacing entity {}", ngsiLdEntity.id) - entityAttributeService.deleteAttributes(entityId) + entityAttributeService.deleteAttributes(entityId, ngsiLdDateTime()).bind() - replaceEntityPayload(ngsiLdEntity, expandedEntity, replacedAt, sub).bind() + val replacedAt = ngsiLdDateTime() + replaceEntityPayload(ngsiLdEntity, expandedEntity, replacedAt).bind() + scopeService.replace(ngsiLdEntity, replacedAt, sub).bind() entityAttributeService.createAttributes( ngsiLdEntity, expandedEntity, @@ -199,8 +221,7 @@ class EntityService( suspend fun replaceEntityPayload( ngsiLdEntity: NgsiLdEntity, expandedEntity: ExpandedEntity, - replacedAt: ZonedDateTime, - sub: Sub? = null + replacedAt: ZonedDateTime ): Either = either { val specificAccessPolicy = ngsiLdEntity.getSpecificAccessPolicy()?.bind() val createdAt = retrieveCreatedAt(ngsiLdEntity.id).bind() @@ -225,9 +246,6 @@ class EntityService( .bind("payload", Json.of(serializedPayload)) .bind("specific_access_policy", specificAccessPolicy?.toString()) .execute() - .map { - scopeService.replaceHistoryEntry(ngsiLdEntity, createdAt, sub) - } } private suspend fun retrieveCreatedAt(entityId: URI): Either = @@ -439,7 +457,7 @@ class EntityService( expandedAttributes: ExpandedAttributes, sub: Sub? = null ): Either = either { - val createdAt = ZonedDateTime.now(ZoneOffset.UTC) + val createdAt = ngsiLdDateTime() expandedAttributes.forEach { (attributeName, expandedAttributeInstances) -> expandedAttributeInstances.forEach { expandedAttributeInstance -> val jsonLdAttribute = mapOf(attributeName to listOf(expandedAttributeInstance)) @@ -545,35 +563,57 @@ class EntityService( } @Transactional - suspend fun upsertEntityPayload(entityId: URI, payload: String): Either = - databaseClient.sql( + suspend fun deleteEntity(entityId: URI, sub: Sub? = null): Either = either { + entityQueryService.checkEntityExistence(entityId).bind() + authorizationService.userCanAdminEntity(entityId, sub.toOption()).bind() + + val deletedAt = ngsiLdDateTime() + val entity = deleteEntityPayload(entityId, deletedAt).bind() + entityAttributeService.deleteAttributes(entityId, deletedAt).bind() + scopeService.addHistoryEntry(entityId, emptyList(), TemporalProperty.DELETED_AT, deletedAt, sub).bind() + + entityEventService.publishEntityDeleteEvent(sub, entity) + } + + @Transactional + suspend fun deleteEntityPayload(entityId: URI, deletedAt: ZonedDateTime): Either = either { + val expandedDeletedEntity = Entity.toExpandedDeletedEntity(entityId, deletedAt) + val entity = databaseClient.sql( """ - INSERT INTO entity_payload (entity_id, payload) - VALUES (:entity_id, :payload) - ON CONFLICT (entity_id) - DO UPDATE SET payload = :payload + UPDATE entity_payload + SET deleted_at = :deleted_at, + payload = :payload, + scopes = null, + specific_access_policy = null, + types = '{}' + WHERE entity_id = :entity_id + RETURNING * """.trimIndent() ) - .bind("payload", Json.of(payload)) .bind("entity_id", entityId) - .execute() + .bind("deleted_at", deletedAt) + .bind("payload", Json.of(serializeObject(expandedDeletedEntity.members))) + .oneToResult { + it.rowToEntity() + } + .bind() + entity + } @Transactional - suspend fun deleteEntity(entityId: URI, sub: Sub? = null): Either = either { - entityQueryService.checkEntityExistence(entityId).bind() + suspend fun permanentlyDeleteEntity(entityId: URI, sub: Sub? = null): Either = either { + entityQueryService.checkEntityExistence(entityId, true).bind() authorizationService.userCanAdminEntity(entityId, sub.toOption()).bind() - val entity = deleteEntityPayload(entityId).bind() - - entityAttributeService.deleteAttributes(entityId).bind() - scopeService.deleteHistory(entityId).bind() + val entity = permanentyDeleteEntityPayload(entityId).bind() + entityAttributeService.permanentlyDeleteAttributes(entityId).bind() authorizationService.removeRightsOnEntity(entityId).bind() entityEventService.publishEntityDeleteEvent(sub, entity) } @Transactional - suspend fun deleteEntityPayload(entityId: URI): Either = either { + suspend fun permanentyDeleteEntityPayload(entityId: URI): Either = either { val entity = databaseClient.sql( """ DELETE FROM entity_payload @@ -612,7 +652,8 @@ class EntityService( entityId, attributeName, datasetId, - deleteAll + deleteAll, + ngsiLdDateTime() ).bind() } updateState( @@ -629,4 +670,36 @@ class EntityService( deleteAll ) } + + @Transactional + suspend fun permanentlyDeleteAttribute( + entityId: URI, + attributeName: ExpandedTerm, + datasetId: URI?, + deleteAll: Boolean = false, + sub: Sub? = null + ): Either = either { + authorizationService.userCanUpdateEntity(entityId, sub.toOption()).bind() + + if (attributeName == NGSILD_SCOPE_PROPERTY) { + scopeService.permanentlyDelete(entityId).bind() + } else { + entityAttributeService.checkEntityAndAttributeExistence( + entityId, + attributeName, + datasetId + ).bind() + entityAttributeService.permanentlyDeleteAttribute( + entityId, + attributeName, + datasetId, + deleteAll + ).bind() + } + updateState( + entityId, + ngsiLdDateTime(), + entityAttributeService.getForEntity(entityId, emptySet(), emptySet()) + ).bind() + } } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/util/AttributeUtils.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/util/AttributeUtils.kt index 32d2c2c35..af8e883d6 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/util/AttributeUtils.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/util/AttributeUtils.kt @@ -7,6 +7,7 @@ import arrow.core.right import com.egm.stellio.search.common.util.deserializeAsMap import com.egm.stellio.search.common.util.valueToDoubleOrNull import com.egm.stellio.search.entity.model.Attribute +import com.egm.stellio.search.entity.model.Attribute.AttributeType import com.egm.stellio.search.entity.model.AttributeMetadata import com.egm.stellio.shared.model.APIException import com.egm.stellio.shared.model.BadRequestDataException @@ -20,12 +21,17 @@ import com.egm.stellio.shared.model.NgsiLdPropertyInstance import com.egm.stellio.shared.model.NgsiLdRelationshipInstance import com.egm.stellio.shared.model.NgsiLdVocabPropertyInstance import com.egm.stellio.shared.model.WKTCoordinates +import com.egm.stellio.shared.model.getMemberValue import com.egm.stellio.shared.model.getPropertyValue +import com.egm.stellio.shared.model.getRelationshipId import com.egm.stellio.shared.util.JsonLdUtils +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_LANGUAGEPROPERTY_VALUE +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_NULL import com.egm.stellio.shared.util.JsonUtils import com.egm.stellio.shared.util.JsonUtils.deserializeAsMap import com.savvasdalkitsis.jsonmerger.JsonMerger import io.r2dbc.postgresql.codec.Json +import java.net.URI import java.time.LocalDate import java.time.LocalTime import java.time.ZonedDateTime @@ -47,35 +53,35 @@ fun NgsiLdAttributeInstance.toAttributeMetadata(): Either guessPropertyValueType(this).let { - Triple(Attribute.AttributeType.Property, it.first, it.second) + Triple(AttributeType.Property, it.first, it.second) } is NgsiLdRelationshipInstance -> Triple( - Attribute.AttributeType.Relationship, + AttributeType.Relationship, Attribute.AttributeValueType.URI, Triple(this.objectId.toString(), null, null) ) is NgsiLdGeoPropertyInstance -> Triple( - Attribute.AttributeType.GeoProperty, + AttributeType.GeoProperty, Attribute.AttributeValueType.GEOMETRY, Triple(null, null, this.coordinates) ) is NgsiLdJsonPropertyInstance -> Triple( - Attribute.AttributeType.JsonProperty, + AttributeType.JsonProperty, Attribute.AttributeValueType.JSON, Triple(JsonUtils.serializeObject(this.json), null, null) ) is NgsiLdLanguagePropertyInstance -> Triple( - Attribute.AttributeType.LanguageProperty, + AttributeType.LanguageProperty, Attribute.AttributeValueType.ARRAY, Triple(JsonUtils.serializeObject(this.languageMap), null, null) ) is NgsiLdVocabPropertyInstance -> Triple( - Attribute.AttributeType.VocabProperty, + AttributeType.VocabProperty, Attribute.AttributeValueType.ARRAY, Triple(JsonUtils.serializeObject(this.vocab), null, null) ) @@ -97,17 +103,17 @@ fun NgsiLdAttributeInstance.toAttributeMetadata(): Either + AttributeType.Property -> guessPropertyValueType(expandedAttributeInstance.getPropertyValue()!!).first - Attribute.AttributeType.Relationship -> Attribute.AttributeValueType.URI - Attribute.AttributeType.GeoProperty -> Attribute.AttributeValueType.GEOMETRY - Attribute.AttributeType.JsonProperty -> Attribute.AttributeValueType.JSON - Attribute.AttributeType.LanguageProperty -> Attribute.AttributeValueType.ARRAY - Attribute.AttributeType.VocabProperty -> Attribute.AttributeValueType.ARRAY + AttributeType.Relationship -> Attribute.AttributeValueType.URI + AttributeType.GeoProperty -> Attribute.AttributeValueType.GEOMETRY + AttributeType.JsonProperty -> Attribute.AttributeValueType.JSON + AttributeType.LanguageProperty -> Attribute.AttributeValueType.ARRAY + AttributeType.VocabProperty -> Attribute.AttributeValueType.ARRAY } fun guessPropertyValueType( @@ -131,6 +137,21 @@ fun guessPropertyValueType( else -> Pair(Attribute.AttributeValueType.STRING, Triple(value.toString(), null, null)) } +/** + * Returns whether the expanded attribute instance holds a NGSI-LD Null value + */ +fun hasNgsiLdNullValue( + expandedAttributeInstance: ExpandedAttributeInstance, + attributeType: AttributeType +): Boolean = + if (attributeType == AttributeType.Relationship) { + val value = expandedAttributeInstance.getRelationshipId() + value is URI && value.toString() == NGSILD_NULL + } else { + val value = expandedAttributeInstance.getMemberValue(attributeType.toExpandedValueMember()) + value is String && value == NGSILD_NULL + } + fun Json.toExpandedAttributeInstance(): ExpandedAttributeInstance = this.deserializeAsMap() as ExpandedAttributeInstance @@ -169,7 +190,7 @@ fun mergePatch( ).deserializeAsMap() ) } - } else if (listOf(JsonLdUtils.NGSILD_LANGUAGEPROPERTY_VALUE).contains(attrName)) { + } else if (listOf(NGSILD_LANGUAGEPROPERTY_VALUE).contains(attrName)) { val sourceLangEntries = source[attrName] as List> val targetLangEntries = sourceLangEntries.toMutableList() (attrValue as List>).forEach { langEntry -> diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt index 2968f7cc6..8cb37778b 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt @@ -37,6 +37,8 @@ import com.egm.stellio.shared.util.JsonLdUtils import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_SCOPE_PROPERTY import com.egm.stellio.shared.util.JsonUtils.serializeObject import com.egm.stellio.shared.util.Sub +import com.egm.stellio.shared.util.getSubFromSecurityContext +import com.egm.stellio.shared.util.ngsiLdDateTime import io.r2dbc.postgresql.codec.Json import org.springframework.r2dbc.core.DatabaseClient import org.springframework.r2dbc.core.bind @@ -72,7 +74,7 @@ class ScopeService( suspend fun addHistoryEntry( entityId: URI, scopes: List, - temportalProperty: TemporalProperty, + temporalProperty: TemporalProperty, createdAt: ZonedDateTime, sub: Sub? = null ): Either = @@ -85,7 +87,7 @@ class ScopeService( .bind("entity_id", entityId) .bind("value", scopes.toTypedArray()) .bind("time", createdAt) - .bind("time_property", temportalProperty.toString()) + .bind("time_property", temporalProperty.toString()) .bind("sub", sub) .execute() @@ -341,12 +343,12 @@ class ScopeService( } @Transactional - suspend fun replaceHistoryEntry( + suspend fun replace( ngsiLdEntity: NgsiLdEntity, createdAt: ZonedDateTime, sub: Sub? = null ): Either = either { - deleteHistory(ngsiLdEntity.id).bind() + delete(ngsiLdEntity.id).bind() createHistory(ngsiLdEntity, createdAt, sub).bind() } @@ -364,10 +366,17 @@ class ScopeService( .execute() .bind() - deleteHistory(entityId).bind() + addHistoryEntry( + entityId, + emptyList(), + TemporalProperty.DELETED_AT, + ngsiLdDateTime(), + getSubFromSecurityContext().getOrNull() + ) } - suspend fun deleteHistory(entityId: URI): Either = + @Transactional + suspend fun permanentlyDelete(entityId: URI): Either = databaseClient.sql( """ DELETE FROM scope_history diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/temporal/model/AttributeInstance.kt b/search-service/src/main/kotlin/com/egm/stellio/search/temporal/model/AttributeInstance.kt index d8ab30296..e8d7ae97f 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/temporal/model/AttributeInstance.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/temporal/model/AttributeInstance.kt @@ -85,7 +85,8 @@ data class AttributeInstance private constructor( enum class TemporalProperty(val propertyName: String) { OBSERVED_AT("observedAt"), CREATED_AT("createdAt"), - MODIFIED_AT("modifiedAt"); + MODIFIED_AT("modifiedAt"), + DELETED_AT("deletedAt"); companion object { fun forPropertyName(propertyName: String): TemporalProperty? = diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/temporal/service/AttributeInstanceService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/temporal/service/AttributeInstanceService.kt index 9f1fa6f32..d718bd9ee 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/temporal/service/AttributeInstanceService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/temporal/service/AttributeInstanceService.kt @@ -14,11 +14,13 @@ import com.egm.stellio.search.common.util.toJsonString import com.egm.stellio.search.common.util.toUuid import com.egm.stellio.search.common.util.toZonedDateTime import com.egm.stellio.search.entity.model.Attribute +import com.egm.stellio.search.entity.model.Attribute.AttributeValueType import com.egm.stellio.search.entity.model.AttributeMetadata import com.egm.stellio.search.entity.util.toAttributeMetadata import com.egm.stellio.search.temporal.model.AggregatedAttributeInstanceResult import com.egm.stellio.search.temporal.model.AggregatedAttributeInstanceResult.AggregateResult import com.egm.stellio.search.temporal.model.AttributeInstance +import com.egm.stellio.search.temporal.model.AttributeInstance.TemporalProperty.DELETED_AT import com.egm.stellio.search.temporal.model.AttributeInstance.TemporalProperty.OBSERVED_AT import com.egm.stellio.search.temporal.model.AttributeInstanceResult import com.egm.stellio.search.temporal.model.FullAttributeInstanceResult @@ -36,6 +38,7 @@ import com.egm.stellio.shared.model.ResourceNotFoundException import com.egm.stellio.shared.model.toNgsiLdAttribute import com.egm.stellio.shared.util.INCONSISTENT_VALUES_IN_AGGREGATION_MESSAGE import com.egm.stellio.shared.util.attributeOrInstanceNotFoundMessage +import com.egm.stellio.shared.util.getSubFromSecurityContext import com.egm.stellio.shared.util.ngsiLdDateTime import org.springframework.r2dbc.core.DatabaseClient import org.springframework.r2dbc.core.bind @@ -123,7 +126,7 @@ class AttributeInstanceService( } @Transactional - suspend fun addAttributeInstance( + suspend fun addObservedAttributeInstance( attributeUuid: UUID, attributeMetadata: AttributeMetadata, attributeValues: Map> @@ -137,6 +140,23 @@ class AttributeInstanceService( return create(attributeInstance) } + @Transactional + suspend fun addDeletedAttributeInstance( + attributeUuid: UUID, + value: String, + deletedAt: ZonedDateTime, + attributeValues: Map> + ): Either { + val attributeInstance = AttributeInstance( + attributeUuid = attributeUuid, + timeAndProperty = deletedAt to DELETED_AT, + value = Triple(value, null, null), + payload = attributeValues, + sub = getSubFromSecurityContext().getOrNull() + ) + return create(attributeInstance) + } + suspend fun search( temporalEntitiesQuery: TemporalEntitiesQuery, attribute: Attribute, @@ -248,9 +268,11 @@ class AttributeInstanceService( } else "SELECT temporal_entity_attribute, min(time) as start, max(time) as end, $allAggregates " } else { - val valueColumn = when (attributes[0].attributeValueType) { - Attribute.AttributeValueType.NUMBER -> "measured_value as value" - Attribute.AttributeValueType.GEOMETRY -> "public.ST_AsText(geo_value) as value" + val valueColumn = when { + // for deletedAt, the NGSI-LD Null representation is always stored as string in value column + temporalQuery.timeproperty == DELETED_AT -> "value" + attributes[0].attributeValueType == AttributeValueType.NUMBER -> "measured_value as value" + attributes[0].attributeValueType == AttributeValueType.GEOMETRY -> "public.ST_AsText(geo_value) as value" else -> "value" } val subColumn = diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/temporal/service/TemporalQueryService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/temporal/service/TemporalQueryService.kt index 2371180c1..52cf57f83 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/temporal/service/TemporalQueryService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/temporal/service/TemporalQueryService.kt @@ -48,7 +48,7 @@ class TemporalQueryService( val attrs = temporalEntitiesQuery.entitiesQuery.attrs val datasetIds = temporalEntitiesQuery.entitiesQuery.datasetId - val attributes = entityAttributeService.getForEntity(entityId, attrs, datasetIds).let { + val attributes = entityAttributeService.getForEntity(entityId, attrs, datasetIds, false).let { if (it.isEmpty()) ResourceNotFoundException( entityOrAttrsNotFoundMessage(entityId.toString(), temporalEntitiesQuery.entitiesQuery.attrs) @@ -94,10 +94,10 @@ class TemporalQueryService( // - timeAt if it is provided // - the oldest value if not (timeAt is optional if querying a temporal entity by id) - if (!temporalEntitiesQuery.withAggregatedValues) - return null + return if (!temporalEntitiesQuery.withAggregatedValues) + null else if (temporalQuery.timeAt != null) - return temporalQuery.timeAt + temporalQuery.timeAt else { val originForAttributes = attributeInstanceService.selectOldestDate(temporalQuery, attributes) @@ -108,7 +108,7 @@ class TemporalQueryService( scopeService.selectOldestDate(entityId, temporalEntitiesQuery.temporalQuery.timeproperty) else null - return when { + when { originForAttributes == null -> originForScope originForScope == null -> originForAttributes else -> minOf(originForAttributes, originForScope) @@ -122,9 +122,11 @@ class TemporalQueryService( ): Either, Int, Range?>> = either { val accessRightFilter = authorizationService.computeAccessRightFilter(sub.toOption()) val attrs = temporalEntitiesQuery.entitiesQuery.attrs - val entitiesIds = entityQueryService.queryEntities(temporalEntitiesQuery.entitiesQuery, accessRightFilter) - val count = entityQueryService.queryEntitiesCount(temporalEntitiesQuery.entitiesQuery, accessRightFilter) - .getOrElse { 0 } + val entitiesIds = + entityQueryService.queryEntities(temporalEntitiesQuery.entitiesQuery, false, accessRightFilter) + val count = + entityQueryService.queryEntitiesCount(temporalEntitiesQuery.entitiesQuery, false, accessRightFilter) + .getOrElse { 0 } // we can have an empty list of entities with a non-zero count (e.g., offset too high) if (entitiesIds.isEmpty()) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/temporal/service/TemporalService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/temporal/service/TemporalService.kt index 25f0939f2..047cfd024 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/temporal/service/TemporalService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/temporal/service/TemporalService.kt @@ -1,6 +1,8 @@ package com.egm.stellio.search.temporal.service import arrow.core.Either +import arrow.core.Either.Left +import arrow.core.Either.Right import arrow.core.raise.either import arrow.core.toOption import com.egm.stellio.search.authorization.service.AuthorizationService @@ -11,12 +13,15 @@ import com.egm.stellio.shared.model.ExpandedAttribute import com.egm.stellio.shared.model.ExpandedAttributes import com.egm.stellio.shared.model.ExpandedEntity import com.egm.stellio.shared.model.ExpandedTerm +import com.egm.stellio.shared.model.NgsiLdEntity import com.egm.stellio.shared.model.addCoreMembers import com.egm.stellio.shared.model.getMemberValueAsDateTime import com.egm.stellio.shared.model.toNgsiLdEntity import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_OBSERVED_AT_PROPERTY import com.egm.stellio.shared.util.Sub +import com.egm.stellio.shared.util.ngsiLdDateTime import org.springframework.stereotype.Service +import org.springframework.transaction.annotation.Transactional import java.net.URI @Service @@ -34,25 +39,28 @@ class TemporalService( jsonLdTemporalEntity: ExpandedEntity, sub: Sub? = null ): Either = either { - val entityDoesNotExist = entityQueryService.checkEntityExistence(entityId, true).isRight() - - if (entityDoesNotExist) { - createTemporalEntity( - entityId, - jsonLdTemporalEntity, - jsonLdTemporalEntity.getAttributes().sorted(), - sub - ).bind() - - CreateOrUpdateResult.CREATED - } else { - upsertTemporalEntity( - entityId, - jsonLdTemporalEntity.getAttributes().sorted(), - sub - ).bind() - - CreateOrUpdateResult.UPSERTED + entityQueryService.isMarkedAsDeleted(entityId).let { + when (it) { + is Left -> { + createTemporalEntity( + entityId, + jsonLdTemporalEntity, + jsonLdTemporalEntity.getAttributes().sorted(), + sub + ).bind() + CreateOrUpdateResult.CREATED + } + is Right -> { + upsertTemporalEntity( + entityId, + jsonLdTemporalEntity, + jsonLdTemporalEntity.getAttributes().sorted(), + it.value, + sub + ).bind() + CreateOrUpdateResult.UPSERTED + } + } } } @@ -64,14 +72,7 @@ class TemporalService( ): Either = either { authorizationService.userCanCreateEntities(sub.toOption()).bind() - // create a view of the entity containing only the most recent instance of each attribute - val expandedEntity = ExpandedEntity( - sortedJsonLdInstances - .keepFirstInstances() - .addCoreMembers(jsonLdTemporalEntity.id, jsonLdTemporalEntity.types) - ) - val ngsiLdEntity = expandedEntity.toNgsiLdEntity().bind() - + val (expandedEntity, ngsiLdEntity) = parseExpandedInstances(sortedJsonLdInstances, jsonLdTemporalEntity).bind() entityService.createEntity(ngsiLdEntity, expandedEntity, sub).bind() entityService.upsertAttributes( entityId, @@ -83,10 +84,17 @@ class TemporalService( internal suspend fun upsertTemporalEntity( entityId: URI, + jsonLdTemporalEntity: ExpandedEntity, sortedJsonLdInstances: ExpandedAttributes, + isDeleted: Boolean, sub: Sub? = null ): Either = either { authorizationService.userCanUpdateEntity(entityId, sub.toOption()).bind() + if (isDeleted) { + val (expandedEntity, ngsiLdEntity) = + parseExpandedInstances(sortedJsonLdInstances, jsonLdTemporalEntity).bind() + entityService.createEntityPayload(ngsiLdEntity, expandedEntity, ngsiLdDateTime()).bind() + } entityService.upsertAttributes( entityId, sortedJsonLdInstances, @@ -94,6 +102,21 @@ class TemporalService( ).bind() } + private suspend fun parseExpandedInstances( + sortedJsonLdInstances: ExpandedAttributes, + jsonLdTemporalEntity: ExpandedEntity + ): Either> = either { + // create a view of the entity containing only the most recent instance of each attribute + val expandedEntity = ExpandedEntity( + sortedJsonLdInstances + .keepFirstInstances() + .addCoreMembers(jsonLdTemporalEntity.id, jsonLdTemporalEntity.types) + ) + val ngsiLdEntity = expandedEntity.toNgsiLdEntity().bind() + + Pair(expandedEntity, ngsiLdEntity) + } + private fun ExpandedAttributes.keepFirstInstances(): ExpandedAttributes = this.mapValues { listOf(it.value.first()) } @@ -124,6 +147,25 @@ class TemporalService( ).bind() } + @Transactional + suspend fun deleteEntity( + entityId: URI, + sub: Sub? = null + ): Either = either { + entityService.permanentlyDeleteEntity(entityId, sub).bind() + } + + @Transactional + suspend fun deleteAttribute( + entityId: URI, + attributeName: ExpandedTerm, + datasetId: URI?, + deleteAll: Boolean = false, + sub: Sub? = null + ): Either = either { + entityService.permanentlyDeleteAttribute(entityId, attributeName, datasetId, deleteAll, sub).bind() + } + suspend fun modifyAttributeInstance( entityId: URI, instanceId: URI, diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/temporal/web/TemporalEntityHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/temporal/web/TemporalEntityHandler.kt index 158ff122f..2b176ef34 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/temporal/web/TemporalEntityHandler.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/temporal/web/TemporalEntityHandler.kt @@ -4,7 +4,6 @@ import arrow.core.Either import arrow.core.left import arrow.core.raise.either import arrow.core.right -import com.egm.stellio.search.entity.service.EntityService import com.egm.stellio.search.temporal.service.TemporalQueryService import com.egm.stellio.search.temporal.service.TemporalService import com.egm.stellio.search.temporal.util.composeTemporalEntitiesQueryFromGet @@ -64,7 +63,6 @@ import java.net.URI class TemporalEntityHandler( private val temporalService: TemporalService, private val temporalQueryService: TemporalQueryService, - private val entityService: EntityService, private val applicationProperties: ApplicationProperties ) : BaseHandler() { @@ -277,7 +275,7 @@ class TemporalEntityHandler( ): ResponseEntity<*> = either { val sub = getSubFromSecurityContext() - entityService.deleteEntity(entityId, sub.getOrNull()).bind() + temporalService.deleteEntity(entityId, sub.getOrNull()).bind() ResponseEntity.status(HttpStatus.NO_CONTENT).build() }.fold( @@ -307,7 +305,7 @@ class TemporalEntityHandler( attrId.checkNameIsNgsiLdSupported().bind() val expandedAttrId = expandJsonLdTerm(attrId, contexts) - entityService.deleteAttribute( + temporalService.deleteAttribute( entityId, expandedAttrId, datasetId, diff --git a/search-service/src/main/resources/db/migration/V0_45__add_deleted_at_temporal_property.sql b/search-service/src/main/resources/db/migration/V0_45__add_deleted_at_temporal_property.sql new file mode 100644 index 000000000..92f72d914 --- /dev/null +++ b/search-service/src/main/resources/db/migration/V0_45__add_deleted_at_temporal_property.sql @@ -0,0 +1,13 @@ +ALTER TABLE temporal_entity_attribute + ADD COLUMN deleted_at timestamp with time zone; + +ALTER TABLE entity_payload + ADD COLUMN deleted_at timestamp with time zone; + +DROP INDEX IF EXISTS temporal_entity_attribute_null_datasetid_uniqueness; + +ALTER TABLE temporal_entity_attribute + DROP CONSTRAINT temporal_entity_attribute_uniqueness; + +ALTER TABLE temporal_entity_attribute + ADD CONSTRAINT temporal_entity_attribute_uniqueness UNIQUE NULLS NOT DISTINCT (entity_id, attribute_name, dataset_id); diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/authorization/listener/IAMListenerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/authorization/listener/IAMListenerTests.kt index 0132af224..cd52b2eb6 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/authorization/listener/IAMListenerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/authorization/listener/IAMListenerTests.kt @@ -348,7 +348,9 @@ class IAMListenerTests { coEvery { entityAccessRightsService.getEntitiesIdsOwnedBySubject("6ad19fe0-fc11-4024-85f2-931c6fa6f7e0") } returns listOf(entityId).right() - coEvery { entityService.deleteEntity(entityId, "6ad19fe0-fc11-4024-85f2-931c6fa6f7e0") } returns Unit.right() + coEvery { + entityService.permanentlyDeleteEntity(entityId, "6ad19fe0-fc11-4024-85f2-931c6fa6f7e0") + } returns Unit.right() iamListener.dispatchIamMessage(subjectDeleteEvent) @@ -360,7 +362,7 @@ class IAMListenerTests { ) } coVerify { - entityService.deleteEntity( + entityService.permanentlyDeleteEntity( eq(entityId), eq("6ad19fe0-fc11-4024-85f2-931c6fa6f7e0") ) diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/AuthorizationServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/AuthorizationServiceTests.kt index 0fd348842..12c3ce2b8 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/AuthorizationServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/AuthorizationServiceTests.kt @@ -40,6 +40,7 @@ class AuthorizationServiceTests { paginationQuery = PaginationQuery(limit = 0, offset = 0), contexts = listOf(applicationProperties.contexts.core) ), + false, listOf(applicationProperties.contexts.core), None ).shouldSucceedWith { diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/EnabledAuthorizationServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/EnabledAuthorizationServiceTests.kt index 2d0cc8d69..edab33f63 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/EnabledAuthorizationServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/EnabledAuthorizationServiceTests.kt @@ -347,7 +347,7 @@ class EnabledAuthorizationServiceTests { @Test fun `it should returned serialized access control entities with a count`() = runTest { coEvery { - entityAccessRightsService.getSubjectAccessRights(any(), any(), any(), any(), any()) + entityAccessRightsService.getSubjectAccessRights(any(), any(), any(), any()) } returns listOf( EntityAccessRights( id = entityId01, @@ -356,7 +356,7 @@ class EnabledAuthorizationServiceTests { ) ).right() coEvery { - entityAccessRightsService.getSubjectAccessRightsCount(any(), any(), any(), any()) + entityAccessRightsService.getSubjectAccessRightsCount(any(), any(), any(), any(), any()) } returns Either.Right(1) coEvery { entityAccessRightsService.getAccessRightsForEntities(any(), any()) @@ -369,6 +369,7 @@ class EnabledAuthorizationServiceTests { contexts = APIC_COMPOUND_CONTEXTS ), contexts = APIC_COMPOUND_CONTEXTS, + includeDeleted = false, sub = Some(subjectUuid) ).shouldSucceedWith { assertEquals(1, it.first) @@ -389,7 +390,7 @@ class EnabledAuthorizationServiceTests { @Test fun `it should returned serialized access control entities with other rigths if user is admin`() = runTest { coEvery { - entityAccessRightsService.getSubjectAccessRights(any(), any(), any(), any(), any()) + entityAccessRightsService.getSubjectAccessRights(any(), any(), any(), any()) } returns listOf( EntityAccessRights( id = entityId01, @@ -403,7 +404,7 @@ class EnabledAuthorizationServiceTests { ) ).right() coEvery { - entityAccessRightsService.getSubjectAccessRightsCount(any(), any(), any(), any()) + entityAccessRightsService.getSubjectAccessRightsCount(any(), any(), any(), any(), any()) } returns Either.Right(1) coEvery { entityAccessRightsService.getAccessRightsForEntities(any(), any()) @@ -424,6 +425,7 @@ class EnabledAuthorizationServiceTests { paginationQuery = PaginationQuery(limit = 10, offset = 0), contexts = APIC_COMPOUND_CONTEXTS ), + includeDeleted = false, contexts = APIC_COMPOUND_CONTEXTS, sub = Some(subjectUuid) ).shouldSucceedWith { @@ -446,10 +448,10 @@ class EnabledAuthorizationServiceTests { @Test fun `it should return serialized access control entities with other rigths if user is owner`() = runTest { coEvery { - entityAccessRightsService.getSubjectAccessRights(any(), any(), any(), any(), any()) + entityAccessRightsService.getSubjectAccessRights(any(), any(), any(), any()) } returns listOf(EntityAccessRights(id = entityId01, types = listOf(BEEHIVE_TYPE), right = IS_OWNER)).right() coEvery { - entityAccessRightsService.getSubjectAccessRightsCount(any(), any(), any(), any()) + entityAccessRightsService.getSubjectAccessRightsCount(any(), any(), any(), any(), any()) } returns Either.Right(1) coEvery { entityAccessRightsService.getAccessRightsForEntities(any(), any()) @@ -476,6 +478,7 @@ class EnabledAuthorizationServiceTests { paginationQuery = PaginationQuery(limit = 10, offset = 0), contexts = APIC_COMPOUND_CONTEXTS ), + includeDeleted = false, contexts = AUTHZ_TEST_COMPOUND_CONTEXTS, sub = Some(subjectUuid) ).shouldSucceedWith { diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/EntityAccessRightsServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/EntityAccessRightsServiceTests.kt index 6365035a4..15a71eecb 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/EntityAccessRightsServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/EntityAccessRightsServiceTests.kt @@ -7,8 +7,10 @@ import com.egm.stellio.search.authorization.getSubjectInfoForGroup import com.egm.stellio.search.authorization.getSubjectInfoForUser import com.egm.stellio.search.authorization.model.SubjectAccessRight import com.egm.stellio.search.authorization.model.SubjectReferential +import com.egm.stellio.search.entity.model.EntitiesQueryFromGet import com.egm.stellio.search.entity.model.Entity import com.egm.stellio.search.entity.service.EntityService +import com.egm.stellio.search.support.WithKafkaContainer import com.egm.stellio.search.support.WithTimescaleContainer import com.egm.stellio.search.support.buildSapAttribute import com.egm.stellio.shared.model.AccessDeniedException @@ -58,7 +60,7 @@ import java.util.UUID @SpringBootTest @ActiveProfiles("test") -class EntityAccessRightsServiceTests : WithTimescaleContainer { +class EntityAccessRightsServiceTests : WithTimescaleContainer, WithKafkaContainer() { @Autowired private lateinit var entityAccessRightsService: EntityAccessRightsService @@ -245,7 +247,10 @@ class EntityAccessRightsServiceTests : WithTimescaleContainer { entityAccessRightsService.getSubjectAccessRights( Some(userUuid), emptyList(), - paginationQuery = PaginationQuery(limit = 100, offset = 0) + entitiesQuery = EntitiesQueryFromGet( + paginationQuery = PaginationQuery(limit = 100, offset = 0), + contexts = emptyList() + ) ).shouldSucceedWith { assertEquals(1, it.size) val entityAccessControl = it[0] @@ -274,7 +279,10 @@ class EntityAccessRightsServiceTests : WithTimescaleContainer { entityAccessRightsService.getSubjectAccessRights( Some(userUuid), emptyList(), - paginationQuery = PaginationQuery(limit = 100, offset = 0) + entitiesQuery = EntitiesQueryFromGet( + paginationQuery = PaginationQuery(limit = 100, offset = 0), + contexts = emptyList() + ) ).shouldSucceedWith { assertEquals(1, it.size) val entityAccessControl = it[0] @@ -305,7 +313,10 @@ class EntityAccessRightsServiceTests : WithTimescaleContainer { entityAccessRightsService.getSubjectAccessRights( Some(userUuid), emptyList(), - paginationQuery = PaginationQuery(limit = 100, offset = 0) + entitiesQuery = EntitiesQueryFromGet( + paginationQuery = PaginationQuery(limit = 100, offset = 0), + contexts = emptyList() + ) ).shouldSucceedWith { assertEquals(2, it.size) it.forEach { entityAccessControl -> @@ -336,8 +347,11 @@ class EntityAccessRightsServiceTests : WithTimescaleContainer { entityAccessRightsService.getSubjectAccessRights( Some(userUuid), emptyList(), - BEEHIVE_TYPE, - paginationQuery = PaginationQuery(limit = 100, offset = 0) + entitiesQuery = EntitiesQueryFromGet( + typeSelection = BEEHIVE_TYPE, + paginationQuery = PaginationQuery(limit = 100, offset = 0), + contexts = emptyList() + ) ).shouldSucceedWith { assertEquals(1, it.size) val entityAccessControl = it[0] @@ -369,9 +383,11 @@ class EntityAccessRightsServiceTests : WithTimescaleContainer { entityAccessRightsService.getSubjectAccessRights( Some(userUuid), emptyList(), - null, - setOf(entityId01, entityId02), - paginationQuery = PaginationQuery(limit = 100, offset = 0) + entitiesQuery = EntitiesQueryFromGet( + ids = setOf(entityId01, entityId02), + paginationQuery = PaginationQuery(limit = 100, offset = 0), + contexts = emptyList() + ) ).shouldSucceedWith { assertEquals(1, it.size) assertEquals(entityId01, it[0].id) @@ -402,9 +418,12 @@ class EntityAccessRightsServiceTests : WithTimescaleContainer { entityAccessRightsService.getSubjectAccessRights( Some(userUuid), emptyList(), - BEEHIVE_TYPE, - setOf(entityId01, entityId03), - paginationQuery = PaginationQuery(limit = 100, offset = 0) + entitiesQuery = EntitiesQueryFromGet( + ids = setOf(entityId01, entityId02), + typeSelection = BEEHIVE_TYPE, + paginationQuery = PaginationQuery(limit = 100, offset = 0), + contexts = emptyList() + ) ).shouldSucceedWith { assertEquals(1, it.size) assertEquals(entityId01, it[0].id) @@ -435,7 +454,10 @@ class EntityAccessRightsServiceTests : WithTimescaleContainer { entityAccessRightsService.getSubjectAccessRights( Some(userUuid), listOf(AccessRight.CAN_WRITE), - paginationQuery = PaginationQuery(limit = 100, offset = 0) + entitiesQuery = EntitiesQueryFromGet( + paginationQuery = PaginationQuery(limit = 100, offset = 0), + contexts = emptyList() + ) ).shouldSucceedWith { assertEquals(1, it.size) val entityAccessControl = it[0] @@ -464,8 +486,11 @@ class EntityAccessRightsServiceTests : WithTimescaleContainer { entityAccessRightsService.getSubjectAccessRights( Some(userUuid), listOf(AccessRight.CAN_WRITE), - "$BEEHIVE_TYPE,$APIARY_TYPE", - paginationQuery = PaginationQuery(limit = 100, offset = 0) + entitiesQuery = EntitiesQueryFromGet( + typeSelection = "$BEEHIVE_TYPE,$APIARY_TYPE", + paginationQuery = PaginationQuery(limit = 100, offset = 0), + contexts = emptyList() + ) ).shouldSucceedWith { assertEquals(1, it.size) val entityAccessControl = it[0] @@ -493,8 +518,11 @@ class EntityAccessRightsServiceTests : WithTimescaleContainer { entityAccessRightsService.getSubjectAccessRights( Some(userUuid), emptyList(), - BEEHIVE_TYPE, - paginationQuery = PaginationQuery(limit = 100, offset = 0) + entitiesQuery = EntitiesQueryFromGet( + typeSelection = BEEHIVE_TYPE, + paginationQuery = PaginationQuery(limit = 100, offset = 0), + contexts = emptyList() + ) ).shouldSucceedWith { assertEquals(1, it.size) val entityAccessControl = it[0] @@ -503,6 +531,66 @@ class EntityAccessRightsServiceTests : WithTimescaleContainer { } } + @Test + fun `getSubjectAccessRights should include deleted entities if it is asked for`() = runTest { + createEntityPayload(entityId01, setOf(BEEHIVE_TYPE), AUTH_READ) + entityAccessRightsService.setRoleOnEntity(userUuid, entityId01, AccessRight.CAN_WRITE).shouldSucceed() + + createEntityPayload(entityId02, setOf(BEEHIVE_TYPE)) + entityAccessRightsService.setRoleOnEntity(userUuid, entityId02, AccessRight.CAN_ADMIN).shouldSucceed() + entityService.deleteEntity(entityId02, userUuid).shouldSucceed() + + entityAccessRightsService.getSubjectAccessRights( + Some(userUuid), + emptyList(), + entitiesQuery = EntitiesQueryFromGet( + paginationQuery = PaginationQuery(limit = 100, offset = 0), + contexts = emptyList() + ), + includeDeleted = true + ).shouldSucceedWith { entityAccessRights -> + assertEquals(2, entityAccessRights.size) + assertEquals(1, entityAccessRights.filter { it.isDeleted }.size) + } + + entityAccessRightsService.getSubjectAccessRightsCount( + Some(userUuid), + emptyList(), + includeDeleted = true + ).shouldSucceedWith { + assertEquals(2, it) + } + } + + @Test + fun `getSubjectAccessRights should not include deleted entities if it is not asked for`() = runTest { + createEntityPayload(entityId01, setOf(BEEHIVE_TYPE)) + entityAccessRightsService.setRoleOnEntity(userUuid, entityId01, AccessRight.CAN_WRITE).shouldSucceed() + + createEntityPayload(entityId02, setOf(BEEHIVE_TYPE)) + entityAccessRightsService.setRoleOnEntity(userUuid, entityId02, AccessRight.CAN_ADMIN).shouldSucceed() + entityService.deleteEntity(entityId02, userUuid).shouldSucceed() + + entityAccessRightsService.getSubjectAccessRights( + Some(userUuid), + emptyList(), + entitiesQuery = EntitiesQueryFromGet( + paginationQuery = PaginationQuery(limit = 100, offset = 0), + contexts = emptyList() + ) + ).shouldSucceedWith { entityAccessRights -> + assertEquals(1, entityAccessRights.size) + assertEquals(0, entityAccessRights.filter { it.isDeleted }.size) + } + + entityAccessRightsService.getSubjectAccessRightsCount( + Some(userUuid), + emptyList() + ).shouldSucceedWith { + assertEquals(1, it) + } + } + @Test fun `it should return nothing when list of entities is empty`() = runTest { entityAccessRightsService.getAccessRightsForEntities(Some(userUuid), emptyList()) diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/SubjectReferentialServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/SubjectReferentialServiceTests.kt index 8d09e591a..5ec77a83b 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/SubjectReferentialServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/SubjectReferentialServiceTests.kt @@ -6,6 +6,7 @@ import com.egm.stellio.search.authorization.getSubjectInfoForGroup import com.egm.stellio.search.authorization.getSubjectInfoForUser import com.egm.stellio.search.authorization.model.SubjectReferential import com.egm.stellio.search.support.EMPTY_JSON_PAYLOAD +import com.egm.stellio.search.support.WithKafkaContainer import com.egm.stellio.search.support.WithTimescaleContainer import com.egm.stellio.shared.model.AccessDeniedException import com.egm.stellio.shared.util.ADMIN_ROLES @@ -36,7 +37,7 @@ import java.util.UUID @SpringBootTest @ActiveProfiles("test") -class SubjectReferentialServiceTests : WithTimescaleContainer { +class SubjectReferentialServiceTests : WithTimescaleContainer, WithKafkaContainer() { @Autowired private lateinit var subjectReferentialService: SubjectReferentialService diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/authorization/web/EntityAccessControlHandlerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/authorization/web/EntityAccessControlHandlerTests.kt index 645272abd..fbd990a86 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/authorization/web/EntityAccessControlHandlerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/authorization/web/EntityAccessControlHandlerTests.kt @@ -596,7 +596,7 @@ class EntityAccessControlHandlerTests { @Test fun `get authorized entities should return 200 and the number of results if requested limit is 0`() { coEvery { - authorizationService.getAuthorizedEntities(any(), any(), any()) + authorizationService.getAuthorizedEntities(any(), any(), any(), any()) } returns Pair(3, emptyList()).right() webClient.get() @@ -610,7 +610,7 @@ class EntityAccessControlHandlerTests { @Test fun `get authorized entities should return 200 and empty response if requested offset does not exist`() { coEvery { - authorizationService.getAuthorizedEntities(any(), any(), any()) + authorizationService.getAuthorizedEntities(any(), any(), any(), any()) } returns Pair(0, emptyList()).right() webClient.get() @@ -620,10 +620,44 @@ class EntityAccessControlHandlerTests { .expectBody().json("[]") } + @Test + fun `get authorized entities should not ask for deleted entities if includeDeleted query param is not provided`() { + coEvery { + authorizationService.getAuthorizedEntities(any(), any(), any(), any()) + } returns Pair(0, emptyList()).right() + + webClient.get() + .uri("/ngsi-ld/v1/entityAccessControl/entities") + .exchange() + .expectStatus().isOk + .expectBody().json("[]") + + coVerify { + authorizationService.getAuthorizedEntities(any(), false, any(), any()) + } + } + + @Test + fun `get authorized entities should ask for deleted entities if includeDeleted query param is true`() { + coEvery { + authorizationService.getAuthorizedEntities(any(), any(), any(), any()) + } returns Pair(0, emptyList()).right() + + webClient.get() + .uri("/ngsi-ld/v1/entityAccessControl/entities?includeDeleted=true") + .exchange() + .expectStatus().isOk + .expectBody().json("[]") + + coVerify { + authorizationService.getAuthorizedEntities(any(), true, any(), any()) + } + } + @Test fun `get authorized entities should return entities I have a right on`() = runTest { coEvery { - authorizationService.getAuthorizedEntities(any(), any(), any()) + authorizationService.getAuthorizedEntities(any(), any(), any(), any()) } returns Pair( 2, listOf( @@ -697,7 +731,7 @@ class EntityAccessControlHandlerTests { @Test fun `get authorized entities should return 204 if authentication is not enabled`() { coEvery { - authorizationService.getAuthorizedEntities(any(), any(), any()) + authorizationService.getAuthorizedEntities(any(), any(), any(), any()) } returns Pair(-1, emptyList()).right() webClient.get() diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationServiceTests.kt index 865e11889..9f99f0a4d 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationServiceTests.kt @@ -5,6 +5,7 @@ import com.egm.stellio.search.csr.model.CSRFilters import com.egm.stellio.search.csr.model.ContextSourceRegistration import com.egm.stellio.search.csr.model.ContextSourceRegistration.Companion.notFoundMessage import com.egm.stellio.search.csr.model.Operation +import com.egm.stellio.search.support.WithKafkaContainer import com.egm.stellio.search.support.WithTimescaleContainer import com.egm.stellio.shared.model.AlreadyExistsException import com.egm.stellio.shared.model.ResourceNotFoundException @@ -33,7 +34,7 @@ import java.util.UUID @SpringBootTest @ActiveProfiles("test") @TestPropertySource(properties = ["application.authentication.enabled=false"]) -class ContextSourceRegistrationServiceTests : WithTimescaleContainer { +class ContextSourceRegistrationServiceTests : WithTimescaleContainer, WithKafkaContainer() { @Autowired private lateinit var contextSourceRegistrationService: ContextSourceRegistrationService diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/discovery/service/AttributeServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/discovery/service/AttributeServiceTests.kt index cd43ffb21..5f9508c19 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/discovery/service/AttributeServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/discovery/service/AttributeServiceTests.kt @@ -52,7 +52,7 @@ import java.time.ZoneOffset @SpringBootTest @ActiveProfiles("test") -class AttributeServiceTests : WithTimescaleContainer, WithKafkaContainer { +class AttributeServiceTests : WithTimescaleContainer, WithKafkaContainer() { @Autowired private lateinit var attributeService: AttributeService diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/discovery/service/EntityTypeServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/discovery/service/EntityTypeServiceTests.kt index abae6170b..a052910d4 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/discovery/service/EntityTypeServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/discovery/service/EntityTypeServiceTests.kt @@ -59,7 +59,7 @@ import org.springframework.test.context.ActiveProfiles @SpringBootTest @ActiveProfiles("test") -class EntityTypeServiceTests : WithTimescaleContainer, WithKafkaContainer { +class EntityTypeServiceTests : WithTimescaleContainer, WithKafkaContainer() { @Autowired private lateinit var entityTypeService: EntityTypeService diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityAttributeServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityAttributeServiceTests.kt index f0ede34f6..654d4a8b2 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityAttributeServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityAttributeServiceTests.kt @@ -18,8 +18,10 @@ import com.egm.stellio.shared.util.BEEHIVE_TYPE import com.egm.stellio.shared.util.INCOMING_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DEFAULT_VOCAB +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_NULL import com.egm.stellio.shared.util.JsonLdUtils.expandAttribute import com.egm.stellio.shared.util.JsonUtils.serializeObject +import com.egm.stellio.shared.util.NGSILD_TEST_CORE_CONTEXTS import com.egm.stellio.shared.util.OUTGOING_PROPERTY import com.egm.stellio.shared.util.TEMPERATURE_PROPERTY import com.egm.stellio.shared.util.assertJsonPayloadsAreEqual @@ -56,7 +58,7 @@ import java.time.ZonedDateTime @SpringBootTest @ActiveProfiles("test") -class EntityAttributeServiceTests : WithTimescaleContainer, WithKafkaContainer { +class EntityAttributeServiceTests : WithTimescaleContainer, WithKafkaContainer() { @Autowired @SpykBean @@ -260,6 +262,9 @@ class EntityAttributeServiceTests : WithTimescaleContainer, WithKafkaContainer { val rawEntity = loadSampleData() coEvery { attributeInstanceService.create(any()) } returns Unit.right() + coEvery { + attributeInstanceService.addDeletedAttributeInstance(any(), any(), any(), any()) + } returns Unit.right() entityAttributeService.createAttributes(rawEntity, APIC_COMPOUND_CONTEXTS) .shouldSucceed() @@ -472,11 +477,151 @@ class EntityAttributeServiceTests : WithTimescaleContainer, WithKafkaContainer { } } + @Test + fun `it should delete an attribute in merge operation if its value is NGSI-LD null`() = runTest { + val rawEntity = loadSampleData() + + coEvery { attributeInstanceService.create(any()) } returns Unit.right() + coEvery { + attributeInstanceService.addDeletedAttributeInstance(any(), any(), any(), any()) + } returns Unit.right() + + entityAttributeService.createAttributes(rawEntity, APIC_COMPOUND_CONTEXTS).shouldSucceed() + + val createdAt = ngsiLdDateTime() + val propertyToDelete = loadSampleData("fragments/beehive_mergeAttribute_null.json") + val expandedAttributes = JsonLdUtils.expandAttributes(propertyToDelete, APIC_COMPOUND_CONTEXTS) + val ngsiLdAttributes = expandedAttributes.toMap().toNgsiLdAttributes().shouldSucceedAndResult() + entityAttributeService.mergeAttributes( + beehiveTestCId, + ngsiLdAttributes, + expandedAttributes, + createdAt, + null, + null + ).shouldSucceedWith { updateResult -> + val updatedDetails = updateResult.updated + assertEquals(1, updatedDetails.size) + assertEquals(1, updatedDetails.filter { it.updateOperationResult == UpdateOperationResult.DELETED }.size) + } + + coVerify(exactly = 1) { + attributeInstanceService.addDeletedAttributeInstance( + any(), + NGSILD_NULL, + createdAt, + expandAttribute( + INCOMING_PROPERTY, + """ + { + "type": "Property", + "value": "urn:ngsi-ld:null" + } + """.trimIndent(), + NGSILD_TEST_CORE_CONTEXTS + ).second[0] + ) + } + } + + @Test + fun `it should delete an attribute in update operation if its value is NGSI-LD null`() = runTest { + val rawEntity = loadSampleData() + + coEvery { attributeInstanceService.create(any()) } returns Unit.right() + coEvery { + attributeInstanceService.addDeletedAttributeInstance(any(), any(), any(), any()) + } returns Unit.right() + + entityAttributeService.createAttributes(rawEntity, APIC_COMPOUND_CONTEXTS).shouldSucceed() + + val createdAt = ngsiLdDateTime() + val propertyToDelete = loadSampleData("fragments/beehive_mergeAttribute_null.json") + val expandedAttributes = JsonLdUtils.expandAttributes(propertyToDelete, APIC_COMPOUND_CONTEXTS) + val ngsiLdAttributes = expandedAttributes.toMap().toNgsiLdAttributes().shouldSucceedAndResult() + entityAttributeService.updateAttributes( + beehiveTestCId, + ngsiLdAttributes, + expandedAttributes, + createdAt, + null + ).shouldSucceedWith { updateResult -> + val updatedDetails = updateResult.updated + assertEquals(1, updatedDetails.size) + assertEquals(1, updatedDetails.filter { it.updateOperationResult == UpdateOperationResult.DELETED }.size) + } + + coVerify(exactly = 1) { + attributeInstanceService.addDeletedAttributeInstance( + any(), + NGSILD_NULL, + createdAt, + expandAttribute( + INCOMING_PROPERTY, + """ + { + "type": "Property", + "value": "urn:ngsi-ld:null" + } + """.trimIndent(), + NGSILD_TEST_CORE_CONTEXTS + ).second[0] + ) + } + } + + @Test + fun `it should delete an attribute in partial attribute update operation if its value is NGSI-LD null`() = runTest { + val rawEntity = loadSampleData() + + coEvery { attributeInstanceService.create(any()) } returns Unit.right() + coEvery { + attributeInstanceService.addDeletedAttributeInstance(any(), any(), any(), any()) + } returns Unit.right() + + entityAttributeService.createAttributes(rawEntity, APIC_COMPOUND_CONTEXTS).shouldSucceed() + + val createdAt = ngsiLdDateTime() + val propertyToDelete = loadSampleData("fragments/beehive_mergeAttribute_null_fragment.json") + val expandedAttribute = expandAttribute(INCOMING_PROPERTY, propertyToDelete, APIC_COMPOUND_CONTEXTS) + entityAttributeService.partialUpdateAttribute( + beehiveTestCId, + expandedAttribute, + createdAt, + null + ).shouldSucceedWith { updateResult -> + val updatedDetails = updateResult.updated + assertEquals(1, updatedDetails.size) + assertEquals(1, updatedDetails.filter { it.updateOperationResult == UpdateOperationResult.DELETED }.size) + } + + coVerify(exactly = 1) { + attributeInstanceService.addDeletedAttributeInstance( + any(), + NGSILD_NULL, + createdAt, + expandAttribute( + INCOMING_PROPERTY, + """ + { + "type": "Property", + "value": "urn:ngsi-ld:null" + } + """.trimIndent(), + NGSILD_TEST_CORE_CONTEXTS + ).second[0] + ) + } + } + @Test fun `it should replace an entity attribute`() = runTest { val rawEntity = loadSampleData() coEvery { attributeInstanceService.create(any()) } returns Unit.right() + coEvery { + attributeInstanceService.addDeletedAttributeInstance(any(), any(), any(), any()) + } returns Unit.right() entityAttributeService.createAttributes(rawEntity, APIC_COMPOUND_CONTEXTS) .shouldSucceed() @@ -581,34 +726,57 @@ class EntityAttributeServiceTests : WithTimescaleContainer, WithKafkaContainer { } @Test - fun `it should delete a temporal attribute references`() = runTest { + fun `it should flag and audit a deleted attribute`() = runTest { val rawEntity = loadSampleData("beehive_two_temporal_properties.jsonld") coEvery { attributeInstanceService.create(any()) } returns Unit.right() - coEvery { attributeInstanceService.deleteInstancesOfAttribute(any(), any(), any()) } returns Unit.right() + coEvery { + attributeInstanceService.addDeletedAttributeInstance(any(), any(), any(), any()) + } returns Unit.right() entityAttributeService.createAttributes(rawEntity, APIC_COMPOUND_CONTEXTS) + val deletedAt = ngsiLdDateTime() entityAttributeService.deleteAttribute( beehiveTestDId, INCOMING_PROPERTY, - null + null, + false, + deletedAt ).shouldSucceed() coVerify { - attributeInstanceService.deleteInstancesOfAttribute(eq(beehiveTestDId), eq(INCOMING_PROPERTY), null) + attributeInstanceService.addDeletedAttributeInstance( + any(), + NGSILD_NULL, + deletedAt, + expandAttribute( + INCOMING_PROPERTY, + """ + { + "type": "Property", + "value": "urn:ngsi-ld:null" + } + """.trimIndent(), + NGSILD_TEST_CORE_CONTEXTS + ).second[0] + ) } entityAttributeService.getForEntityAndAttribute(beehiveTestDId, INCOMING_PROPERTY) - .shouldFail { assertInstanceOf(ResourceNotFoundException::class.java, it) } + .shouldSucceedWith { + assertEquals(deletedAt, it.deletedAt) + } } @Test - fun `it should delete references of all temporal attribute instances`() = runTest { + fun `it should flag and audit all instances of a deleted attribute`() = runTest { val rawEntity = loadSampleData("beehive_multi_instance_property.jsonld") coEvery { attributeInstanceService.create(any()) } returns Unit.right() - coEvery { attributeInstanceService.deleteAllInstancesOfAttribute(any(), any()) } returns Unit.right() + coEvery { + attributeInstanceService.addDeletedAttributeInstance(any(), any(), any(), any()) + } returns Unit.right() entityAttributeService.createAttributes(rawEntity, APIC_COMPOUND_CONTEXTS) @@ -616,17 +784,88 @@ class EntityAttributeServiceTests : WithTimescaleContainer, WithKafkaContainer { beehiveTestCId, INCOMING_PROPERTY, null, - deleteAll = true + deleteAll = true, + ngsiLdDateTime() + ).shouldSucceed() + + coVerify(exactly = 2) { + attributeInstanceService.addDeletedAttributeInstance(any(), any(), any(), any()) + } + + entityAttributeService.getForEntityAndAttribute(beehiveTestCId, INCOMING_PROPERTY) + .shouldFail { assertInstanceOf(ResourceNotFoundException::class.java, it) } + } + + @Test + fun `it should flag and audit all deleted attributes of an entity`() = runTest { + val rawEntity = loadSampleData("beehive.jsonld") + + coEvery { attributeInstanceService.create(any()) } returns Unit.right() + coEvery { + attributeInstanceService.addDeletedAttributeInstance(any(), any(), any(), any()) + } returns Unit.right() + + entityAttributeService.createAttributes(rawEntity, APIC_COMPOUND_CONTEXTS) + + entityAttributeService.deleteAttributes( + beehiveTestCId, + ngsiLdDateTime() + ).shouldSucceed() + + coVerify(exactly = 4) { + attributeInstanceService.addDeletedAttributeInstance(any(), any(), any(), any()) + } + + entityAttributeService.getForEntityAndAttribute(beehiveTestCId, INCOMING_PROPERTY) + .shouldSucceedWith { assertTrue(it.deletedAt != null) } + } + + @Test + fun `it should permanently delete an attribute of an entity`() = runTest { + val rawEntity = loadSampleData("beehive.jsonld") + + coEvery { attributeInstanceService.create(any()) } returns Unit.right() + coEvery { attributeInstanceService.deleteInstancesOfAttribute(any(), any(), any()) } returns Unit.right() + + entityAttributeService.createAttributes(rawEntity, APIC_COMPOUND_CONTEXTS) + + entityAttributeService.permanentlyDeleteAttribute( + beehiveTestCId, + INCOMING_PROPERTY, + null, + false ).shouldSucceed() coVerify { - attributeInstanceService.deleteAllInstancesOfAttribute(eq(beehiveTestCId), eq(INCOMING_PROPERTY)) + attributeInstanceService.deleteInstancesOfAttribute(beehiveTestCId, INCOMING_PROPERTY, null) } entityAttributeService.getForEntityAndAttribute(beehiveTestCId, INCOMING_PROPERTY) .shouldFail { assertInstanceOf(ResourceNotFoundException::class.java, it) } } + @Test + fun `it should permanently delete all attribute of an entity`() = runTest { + val rawEntity = loadSampleData("beehive.jsonld") + + coEvery { attributeInstanceService.create(any()) } returns Unit.right() + coEvery { attributeInstanceService.deleteInstancesOfEntity(any()) } returns Unit.right() + + entityAttributeService.createAttributes(rawEntity, APIC_COMPOUND_CONTEXTS) + + entityAttributeService.permanentlyDeleteAttributes(beehiveTestCId).shouldSucceed() + + coVerify { + attributeInstanceService.deleteInstancesOfEntity( + match { it.size == 4 } + ) + } + + entityAttributeService.getForEntity(beehiveTestCId, emptySet(), emptySet()).also { + assertTrue(it.isEmpty()) + } + } + @Test fun `it should return a right unit if entiy and attribute exist`() = runTest { val rawEntity = loadSampleData() diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityOperationServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityOperationServiceTests.kt index 0cbb4a47d..004a15c6f 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityOperationServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityOperationServiceTests.kt @@ -296,7 +296,10 @@ class EntityOperationServiceTests { ) assertTrue(batchOperationResult.errors.isEmpty()) - coVerify { entityService.replaceEntity(firstEntityURI, any(), any(), any()) } + coVerify { + entityService.replaceEntity(firstEntityURI, any(), any(), any()) + entityService.replaceEntity(secondEntityURI, any(), any(), any()) + } } @Test diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityQueryServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityQueryServiceTests.kt index 527bf7d52..c60a3fd4e 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityQueryServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityQueryServiceTests.kt @@ -7,7 +7,6 @@ import com.egm.stellio.search.entity.model.Entity import com.egm.stellio.search.support.WithKafkaContainer import com.egm.stellio.search.support.WithTimescaleContainer import com.egm.stellio.search.support.buildDefaultQueryParams -import com.egm.stellio.shared.model.AlreadyExistsException import com.egm.stellio.shared.model.ResourceNotFoundException import com.egm.stellio.shared.util.AUTHZ_TEST_COMPOUND_CONTEXTS import com.egm.stellio.shared.util.AuthContextModel.SpecificAccessPolicy.AUTH_READ @@ -29,6 +28,7 @@ import kotlinx.coroutines.test.runTest import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired @@ -40,7 +40,7 @@ import java.time.ZonedDateTime @SpringBootTest @ActiveProfiles("test") -class EntityQueryServiceTests : WithTimescaleContainer, WithKafkaContainer { +class EntityQueryServiceTests : WithTimescaleContainer, WithKafkaContainer() { @Autowired private lateinit var entityQueryService: EntityQueryService @@ -96,14 +96,15 @@ class EntityQueryServiceTests : WithTimescaleContainer, WithKafkaContainer { @Test fun `it should return a list of JSON-LD entities when querying entities`() = runTest { + coEvery { authorizationService.userCanCreateEntities(any()) } returns Unit.right() + coEvery { authorizationService.createOwnerRight(any(), any()) } returns Unit.right() coEvery { authorizationService.computeAccessRightFilter(any()) } returns { null } loadAndPrepareSampleData("beehive.jsonld") .map { - entityService.createEntityPayload( + entityService.createEntity( it.second, - it.first, - now, + it.first ).shouldSucceed() } @@ -210,7 +211,7 @@ class EntityQueryServiceTests : WithTimescaleContainer, WithKafkaContainer { } @Test - fun `it should check the existence or non-existence of an entity`() = runTest { + fun `it should check the existence of an entity`() = runTest { loadMinimalEntity(entity01Uri, setOf(BEEHIVE_TYPE)) .sampleDataToNgsiLdEntity() .map { @@ -224,8 +225,25 @@ class EntityQueryServiceTests : WithTimescaleContainer, WithKafkaContainer { entityQueryService.checkEntityExistence(entity01Uri).shouldSucceed() entityQueryService.checkEntityExistence(entity02Uri) .shouldFail { assert(it is ResourceNotFoundException) } - entityQueryService.checkEntityExistence(entity01Uri, true) - .shouldFail { assert(it is AlreadyExistsException) } - entityQueryService.checkEntityExistence(entity02Uri, true).shouldSucceed() + } + + @Test + fun `it should check the state of an entity`() = runTest { + loadMinimalEntity(entity01Uri, setOf(BEEHIVE_TYPE)) + .sampleDataToNgsiLdEntity() + .map { + entityService.createEntityPayload( + it.second, + it.first, + now + ) + } + + entityQueryService.isMarkedAsDeleted(entity01Uri) + .shouldSucceedWith { + assertFalse(it) + } + entityQueryService.isMarkedAsDeleted(entity02Uri) + .shouldFail { assert(it is ResourceNotFoundException) } } } diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityServiceQueriesTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityServiceQueriesTests.kt index 4b6396bd9..f8c69a2aa 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityServiceQueriesTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityServiceQueriesTests.kt @@ -2,6 +2,7 @@ package com.egm.stellio.search.entity.service import arrow.core.right import com.egm.stellio.search.authorization.service.AuthorizationService +import com.egm.stellio.search.entity.model.Attribute import com.egm.stellio.search.entity.model.EntitiesQueryFromGet import com.egm.stellio.search.entity.model.EntitiesQueryFromPost import com.egm.stellio.search.entity.model.Entity @@ -42,6 +43,7 @@ import org.junit.jupiter.params.provider.CsvSource import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest import org.springframework.data.r2dbc.core.R2dbcEntityTemplate +import org.springframework.data.r2dbc.core.delete import org.springframework.data.relational.core.query.Criteria import org.springframework.data.relational.core.query.Query import org.springframework.data.relational.core.query.Update @@ -50,7 +52,7 @@ import java.net.URI @SpringBootTest @ActiveProfiles("test") -class EntityServiceQueriesTests : WithTimescaleContainer, WithKafkaContainer { +class EntityServiceQueriesTests : WithTimescaleContainer, WithKafkaContainer() { @Autowired private lateinit var entityQueryService: EntityQueryService @@ -95,15 +97,8 @@ class EntityServiceQueriesTests : WithTimescaleContainer, WithKafkaContainer { @AfterAll fun deleteEntities() { - coEvery { authorizationService.userCanAdminEntity(any(), any()) } returns Unit.right() - coEvery { authorizationService.removeRightsOnEntity(any()) } returns Unit.right() - runBlocking { - entityService.deleteEntity(entity01Uri) - entityService.deleteEntity(entity02Uri) - entityService.deleteEntity(entity03Uri) - entityService.deleteEntity(entity04Uri) - entityService.deleteEntity(entity05Uri) - } + r2dbcEntityTemplate.delete().from("entity_payload").all().block() + r2dbcEntityTemplate.delete().from("temporal_entity_attribute").all().block() } @ParameterizedTest diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityServiceTests.kt index 5a5b03897..54b86b4c5 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityServiceTests.kt @@ -1,5 +1,6 @@ package com.egm.stellio.search.entity.service +import arrow.core.left import arrow.core.right import arrow.core.toOption import com.egm.stellio.search.authorization.service.AuthorizationService @@ -9,50 +10,57 @@ import com.egm.stellio.search.entity.model.Entity import com.egm.stellio.search.entity.model.UpdateOperationResult import com.egm.stellio.search.entity.model.UpdateResult import com.egm.stellio.search.entity.model.UpdatedDetails -import com.egm.stellio.search.support.EMPTY_PAYLOAD import com.egm.stellio.search.support.WithKafkaContainer import com.egm.stellio.search.support.WithTimescaleContainer +import com.egm.stellio.shared.model.AccessDeniedException +import com.egm.stellio.shared.model.AlreadyExistsException +import com.egm.stellio.shared.model.ResourceNotFoundException import com.egm.stellio.shared.util.APIARY_TYPE import com.egm.stellio.shared.util.APIC_COMPOUND_CONTEXTS import com.egm.stellio.shared.util.BEEHIVE_TYPE import com.egm.stellio.shared.util.INCOMING_PROPERTY +import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_ID import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DEFAULT_VOCAB +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DELETED_AT_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_SCOPE_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.expandAttribute import com.egm.stellio.shared.util.JsonLdUtils.expandAttributes import com.egm.stellio.shared.util.JsonUtils.deserializeExpandedPayload +import com.egm.stellio.shared.util.OUTGOING_PROPERTY import com.egm.stellio.shared.util.loadAndPrepareSampleData import com.egm.stellio.shared.util.loadMinimalEntity import com.egm.stellio.shared.util.loadSampleData import com.egm.stellio.shared.util.ngsiLdDateTime import com.egm.stellio.shared.util.sampleDataToNgsiLdEntity +import com.egm.stellio.shared.util.shouldFail import com.egm.stellio.shared.util.shouldSucceed import com.egm.stellio.shared.util.shouldSucceedAndResult import com.egm.stellio.shared.util.shouldSucceedWith import com.egm.stellio.shared.util.toUri import com.ninjasquad.springmockk.MockkBean +import io.mockk.Called import io.mockk.coEvery import io.mockk.coVerify import kotlinx.coroutines.Job import kotlinx.coroutines.test.runTest +import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertInstanceOf import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.junit.jupiter.api.assertThrows import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest -import org.springframework.dao.DataIntegrityViolationException import org.springframework.data.r2dbc.core.R2dbcEntityTemplate import org.springframework.data.r2dbc.core.delete import org.springframework.test.context.ActiveProfiles @SpringBootTest @ActiveProfiles("test") -class EntityServiceTests : WithTimescaleContainer, WithKafkaContainer { +class EntityServiceTests : WithTimescaleContainer, WithKafkaContainer() { @Autowired private lateinit var entityService: EntityService @@ -145,7 +153,7 @@ class EntityServiceTests : WithTimescaleContainer, WithKafkaContainer { } @Test - fun `it should not create an entity payload if one already existed`() = runTest { + fun `it should not create an entity payload if one already exists`() = runTest { val (jsonLdEntity, ngsiLdEntity) = loadMinimalEntity(entity01Uri, setOf(BEEHIVE_TYPE)).sampleDataToNgsiLdEntity().shouldSucceedAndResult() entityService.createEntityPayload( @@ -154,13 +162,91 @@ class EntityServiceTests : WithTimescaleContainer, WithKafkaContainer { now, ) - assertThrows { - entityService.createEntityPayload( - ngsiLdEntity, - jsonLdEntity, - now, - ) - } + entityService.createEntity( + ngsiLdEntity, + jsonLdEntity, + sub, + ).shouldFail { assertInstanceOf(AlreadyExistsException::class.java, it) } + } + + @Test + fun `it should allow to create an entity over a deleted one if authorized`() = runTest { + coEvery { entityAttributeService.deleteAttributes(any(), any()) } returns Unit.right() + coEvery { authorizationService.userCanAdminEntity(any(), any()) } returns Unit.right() + coEvery { entityAttributeService.createAttributes(any(), any(), any(), any(), any()) } returns Unit.right() + coEvery { authorizationService.createOwnerRight(any(), any()) } returns Unit.right() + + val (expandedEntity, ngsiLdEntity) = + loadMinimalEntity(entity01Uri, setOf(BEEHIVE_TYPE)) + .sampleDataToNgsiLdEntity() + .shouldSucceedAndResult() + + entityService.createEntityPayload( + ngsiLdEntity, + expandedEntity, + now + ) + + entityService.deleteEntityPayload(entity01Uri, ngsiLdDateTime()) + .shouldSucceedWith { + assertEquals(entity01Uri, it.entityId) + assertNotNull(it.payload) + } + + entityService.createEntity(ngsiLdEntity, expandedEntity, null) + .shouldSucceed() + } + + @Test + fun `it should not allow to create an entity over a deleted one if not authorized`() = runTest { + coEvery { entityAttributeService.deleteAttributes(any(), any()) } returns Unit.right() + coEvery { + authorizationService.userCanAdminEntity(any(), any()) + } returns AccessDeniedException("Unauthorized").left() + + val (expandedEntity, ngsiLdEntity) = + loadMinimalEntity(entity01Uri, setOf(BEEHIVE_TYPE)) + .sampleDataToNgsiLdEntity() + .shouldSucceedAndResult() + + entityService.createEntityPayload( + ngsiLdEntity, + expandedEntity, + now + ) + + entityService.deleteEntityPayload(entity01Uri, ngsiLdDateTime()) + .shouldSucceedWith { + assertEquals(entity01Uri, it.entityId) + assertNotNull(it.payload) + } + + entityService.createEntity(ngsiLdEntity, expandedEntity, null) + .shouldFail { assertInstanceOf(AccessDeniedException::class.java, it) } + } + + @Test + fun `it should create the deleted representation of an entity when deleting it`() = runTest { + val (expandedEntity, ngsiLdEntity) = + loadMinimalEntity(entity01Uri, setOf(BEEHIVE_TYPE)) + .sampleDataToNgsiLdEntity() + .shouldSucceedAndResult() + + entityService.createEntityPayload( + ngsiLdEntity, + expandedEntity, + now + ).shouldSucceed() + + entityService.deleteEntityPayload(entity01Uri, ngsiLdDateTime()).shouldSucceed() + + entityQueryService.retrieve(entity01Uri) + .shouldSucceedWith { entity -> + val payload = entity.payload.deserializeAsMap() + assertThat(payload) + .hasSize(2) + .containsKeys(JSONLD_ID, NGSILD_DELETED_AT_PROPERTY) + } } @Test @@ -334,7 +420,7 @@ class EntityServiceTests : WithTimescaleContainer, WithKafkaContainer { coEvery { entityAttributeService.createAttributes(any(), any(), any(), any(), any()) } returns Unit.right() - coEvery { entityAttributeService.deleteAttributes(any()) } returns Unit.right() + coEvery { entityAttributeService.deleteAttributes(any(), any()) } returns Unit.right() coEvery { authorizationService.createOwnerRight(any(), any()) } returns Unit.right() val (expandedEntity, ngsiLdEntity) = @@ -364,7 +450,7 @@ class EntityServiceTests : WithTimescaleContainer, WithKafkaContainer { coVerify { authorizationService.userCanCreateEntities(sub.toOption()) authorizationService.userCanUpdateEntity(beehiveTestCId, sub.toOption()) - entityAttributeService.deleteAttributes(beehiveTestCId) + entityAttributeService.deleteAttributes(beehiveTestCId, any()) entityAttributeService.createAttributes( any(), any(), @@ -459,8 +545,16 @@ class EntityServiceTests : WithTimescaleContainer, WithKafkaContainer { } @Test - fun `it should upsert an entity payload if one already existed`() = runTest { - loadMinimalEntity(entity01Uri, setOf(BEEHIVE_TYPE)) + fun `it should remove the scopes from an entity`() = runTest { + coEvery { authorizationService.userCanUpdateEntity(any(), any()) } returns Unit.right() + coEvery { + entityAttributeService.addAttribute(any(), any(), any(), any(), any(), any()) + } returns Unit.right() + coEvery { + entityAttributeService.getForEntity(any(), any(), any()) + } returns emptyList() + + loadSampleData("beehive_with_scope.jsonld") .sampleDataToNgsiLdEntity() .map { entityService.createEntityPayload( @@ -470,13 +564,20 @@ class EntityServiceTests : WithTimescaleContainer, WithKafkaContainer { ) } - entityService.upsertEntityPayload(entity01Uri, EMPTY_PAYLOAD) + entityService.deleteAttribute(beehiveTestCId, NGSILD_SCOPE_PROPERTY, null) .shouldSucceed() + + entityQueryService.retrieve(beehiveTestCId) + .shouldSucceedWith { + assertNull(it.scopes) + } } @Test - fun `it should delete an entity payload`() = runTest { - coEvery { entityAttributeService.deleteAttributes(any()) } returns Unit.right() + fun `it should permanently delete an entity`() = runTest { + coEvery { authorizationService.userCanAdminEntity(any(), any()) } returns Unit.right() + coEvery { entityAttributeService.permanentlyDeleteAttributes(any()) } returns Unit.right() + coEvery { authorizationService.removeRightsOnEntity(any()) } returns Unit.right() loadMinimalEntity(entity01Uri, setOf(BEEHIVE_TYPE)) .sampleDataToNgsiLdEntity() @@ -488,36 +589,47 @@ class EntityServiceTests : WithTimescaleContainer, WithKafkaContainer { ) } - entityService.deleteEntityPayload(entity01Uri) - .shouldSucceedWith { - assertEquals(entity01Uri, it.entityId) - assertNotNull(it.payload) + entityService.permanentlyDeleteEntity(entity01Uri).shouldSucceed() + + entityQueryService.retrieve(entity01Uri) + .shouldFail { + assertInstanceOf(ResourceNotFoundException::class.java, it) } + } - // if correctly deleted, we should be able to create a new one - loadMinimalEntity(entity01Uri, setOf(BEEHIVE_TYPE)) - .sampleDataToNgsiLdEntity() + @Test + fun `it should permanently delete an attribute`() = runTest { + coEvery { authorizationService.userCanUpdateEntity(any(), any()) } returns Unit.right() + coEvery { entityAttributeService.checkEntityAndAttributeExistence(any(), any(), any()) } returns Unit.right() + coEvery { entityAttributeService.permanentlyDeleteAttribute(any(), any(), any(), any()) } returns Unit.right() + coEvery { entityAttributeService.getForEntity(any(), any(), any()) } returns emptyList() + + loadAndPrepareSampleData("beehive.jsonld") .map { entityService.createEntityPayload( it.second, it.first, now - ).shouldSucceed() + ) } + + entityService.permanentlyDeleteAttribute(beehiveTestCId, INCOMING_PROPERTY, null).shouldSucceed() + + coVerify { + entityAttributeService.checkEntityAndAttributeExistence(beehiveTestCId, INCOMING_PROPERTY, null) + entityAttributeService.permanentlyDeleteAttribute(beehiveTestCId, INCOMING_PROPERTY, null, false) + entityAttributeService.getForEntity(beehiveTestCId, emptySet(), emptySet()) + } } @Test - fun `it should remove the scopes from an entity`() = runTest { + fun `it should return a ResourceNotFound error if trying to permanently delete an unknown attribute`() = runTest { coEvery { authorizationService.userCanUpdateEntity(any(), any()) } returns Unit.right() coEvery { - entityAttributeService.addAttribute(any(), any(), any(), any(), any(), any()) - } returns Unit.right() - coEvery { - entityAttributeService.getForEntity(any(), any(), any()) - } returns emptyList() + entityAttributeService.checkEntityAndAttributeExistence(any(), any(), any()) + } returns ResourceNotFoundException("Entity does not exist").left() - loadSampleData("beehive_with_scope.jsonld") - .sampleDataToNgsiLdEntity() + loadAndPrepareSampleData("beehive.jsonld") .map { entityService.createEntityPayload( it.second, @@ -526,12 +638,16 @@ class EntityServiceTests : WithTimescaleContainer, WithKafkaContainer { ) } - entityService.deleteAttribute(beehiveTestCId, NGSILD_SCOPE_PROPERTY, null) - .shouldSucceed() + entityService.permanentlyDeleteAttribute(beehiveTestCId, OUTGOING_PROPERTY, null).shouldFail { + assertInstanceOf(ResourceNotFoundException::class.java, it) + } - entityQueryService.retrieve(beehiveTestCId) - .shouldSucceedWith { - assertNull(it.scopes) - } + coVerify { + entityAttributeService.checkEntityAndAttributeExistence(beehiveTestCId, OUTGOING_PROPERTY, null) + listOf( + entityAttributeService.permanentlyDeleteAttribute(beehiveTestCId, OUTGOING_PROPERTY, null, false), + entityAttributeService.getForEntity(beehiveTestCId, emptySet(), emptySet()) + ) wasNot Called + } } } diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/entity/util/AttributeUtilsTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/entity/util/AttributeUtilsTests.kt index b9d039538..5c15c6ad4 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/entity/util/AttributeUtilsTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/entity/util/AttributeUtilsTests.kt @@ -1,11 +1,13 @@ package com.egm.stellio.search.entity.util import com.egm.stellio.search.entity.model.Attribute +import com.egm.stellio.search.entity.model.Attribute.AttributeType import com.egm.stellio.shared.util.JsonLdUtils.expandAttribute import com.egm.stellio.shared.util.NGSILD_TEST_CORE_CONTEXTS import com.egm.stellio.shared.util.ngsiLdDateTime import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import org.springframework.test.context.ActiveProfiles import java.net.URI @@ -23,7 +25,7 @@ class AttributeUtilsTests { ) assertEquals( Attribute.AttributeValueType.STRING, - guessAttributeValueType(Attribute.AttributeType.Property, expandedStringProperty.second[0]) + guessAttributeValueType(AttributeType.Property, expandedStringProperty.second[0]) ) } @@ -36,7 +38,7 @@ class AttributeUtilsTests { ) assertEquals( Attribute.AttributeValueType.NUMBER, - guessAttributeValueType(Attribute.AttributeType.Property, expandedBooleanProperty.second[0]) + guessAttributeValueType(AttributeType.Property, expandedBooleanProperty.second[0]) ) } @@ -49,7 +51,7 @@ class AttributeUtilsTests { ) assertEquals( Attribute.AttributeValueType.NUMBER, - guessAttributeValueType(Attribute.AttributeType.Property, expandedBooleanProperty.second[0]) + guessAttributeValueType(AttributeType.Property, expandedBooleanProperty.second[0]) ) } @@ -62,7 +64,7 @@ class AttributeUtilsTests { ) assertEquals( Attribute.AttributeValueType.BOOLEAN, - guessAttributeValueType(Attribute.AttributeType.Property, expandedBooleanProperty.second[0]) + guessAttributeValueType(AttributeType.Property, expandedBooleanProperty.second[0]) ) } @@ -75,7 +77,7 @@ class AttributeUtilsTests { ) assertEquals( Attribute.AttributeValueType.OBJECT, - guessAttributeValueType(Attribute.AttributeType.Property, expandedListProperty.second[0]) + guessAttributeValueType(AttributeType.Property, expandedListProperty.second[0]) ) } @@ -88,7 +90,7 @@ class AttributeUtilsTests { ) assertEquals( Attribute.AttributeValueType.ARRAY, - guessAttributeValueType(Attribute.AttributeType.Property, expandedListProperty.second[0]) + guessAttributeValueType(AttributeType.Property, expandedListProperty.second[0]) ) } @@ -101,7 +103,7 @@ class AttributeUtilsTests { ) assertEquals( Attribute.AttributeValueType.TIME, - guessAttributeValueType(Attribute.AttributeType.Property, expandedTimeProperty.second[0]) + guessAttributeValueType(AttributeType.Property, expandedTimeProperty.second[0]) ) } @@ -114,7 +116,7 @@ class AttributeUtilsTests { ) assertEquals( Attribute.AttributeValueType.DATETIME, - guessAttributeValueType(Attribute.AttributeType.Property, expandedTimeProperty.second[0]) + guessAttributeValueType(AttributeType.Property, expandedTimeProperty.second[0]) ) } @@ -130,7 +132,7 @@ class AttributeUtilsTests { ) assertEquals( Attribute.AttributeValueType.GEOMETRY, - guessAttributeValueType(Attribute.AttributeType.GeoProperty, expandedGeoProperty.second[0]) + guessAttributeValueType(AttributeType.GeoProperty, expandedGeoProperty.second[0]) ) } @@ -146,7 +148,7 @@ class AttributeUtilsTests { ) assertEquals( Attribute.AttributeValueType.JSON, - guessAttributeValueType(Attribute.AttributeType.JsonProperty, expandedJsonProperty.second[0]) + guessAttributeValueType(AttributeType.JsonProperty, expandedJsonProperty.second[0]) ) } @@ -160,9 +162,79 @@ class AttributeUtilsTests { assertEquals( Attribute.AttributeValueType.URI, guessAttributeValueType( - Attribute.AttributeType.Relationship, + AttributeType.Relationship, expandedGeoRelationship.second[0] ) ) } + + @Test + fun `it should find a Property whose value is NGSI-LD Null`() = runTest { + val expandedProperty = expandAttribute( + """ + { + "property": { + "type": "Property", + "value": "urn:ngsi-ld:null" + } + } + """.trimIndent(), + NGSILD_TEST_CORE_CONTEXTS + ).second[0] + + assertTrue(hasNgsiLdNullValue(expandedProperty, AttributeType.Property)) + } + + @Test + fun `it should find a LanguageProperty whose value is NGSI-LD Null`() = runTest { + val expandedProperty = expandAttribute( + """ + { + "langProperty": { + "type": "LanguageProperty", + "languageMap": { + "@none": "urn:ngsi-ld:null" + } + } + } + """.trimIndent(), + NGSILD_TEST_CORE_CONTEXTS + ).second[0] + + assertTrue(hasNgsiLdNullValue(expandedProperty, AttributeType.LanguageProperty)) + } + + @Test + fun `it should find a JsonProperty whose value is NGSI-LD Null`() = runTest { + val expandedProperty = expandAttribute( + """ + { + "jsonProperty": { + "type": "JsonProperty", + "json": "urn:ngsi-ld:null" + } + } + """.trimIndent(), + NGSILD_TEST_CORE_CONTEXTS + ).second[0] + + assertTrue(hasNgsiLdNullValue(expandedProperty, AttributeType.JsonProperty)) + } + + @Test + fun `it should find a Relationship whose value is NGSI-LD Null`() = runTest { + val expandedProperty = expandAttribute( + """ + { + "relationship": { + "type": "Relationship", + "object": "urn:ngsi-ld:null" + } + } + """.trimIndent(), + NGSILD_TEST_CORE_CONTEXTS + ).second[0] + + assertTrue(hasNgsiLdNullValue(expandedProperty, AttributeType.Relationship)) + } } diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/scope/ScopeServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/scope/ScopeServiceTests.kt index dca484c85..24bc39f48 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/scope/ScopeServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/scope/ScopeServiceTests.kt @@ -23,7 +23,6 @@ import com.egm.stellio.shared.util.shouldSucceed import com.egm.stellio.shared.util.shouldSucceedAndResult import com.egm.stellio.shared.util.shouldSucceedWith import com.egm.stellio.shared.util.toUri -import kotlinx.coroutines.runBlocking import kotlinx.coroutines.test.runTest import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.AfterEach @@ -45,7 +44,7 @@ import java.util.stream.Stream @SpringBootTest @ActiveProfiles("test") -class ScopeServiceTests : WithTimescaleContainer, WithKafkaContainer { +class ScopeServiceTests : WithTimescaleContainer, WithKafkaContainer() { @Autowired private lateinit var scopeService: ScopeService @@ -64,10 +63,6 @@ class ScopeServiceTests : WithTimescaleContainer, WithKafkaContainer { @AfterEach fun clearEntityPayloadTable() { r2dbcEntityTemplate.delete().from("entity_payload").all().block() - - runBlocking { - scopeService.delete(beehiveTestCId) - } } @Suppress("unused") diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/support/WithKafkaContainer.kt b/search-service/src/test/kotlin/com/egm/stellio/search/support/WithKafkaContainer.kt index f8f9fe1f2..727ef5444 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/support/WithKafkaContainer.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/support/WithKafkaContainer.kt @@ -1,29 +1,21 @@ package com.egm.stellio.search.support -import org.springframework.test.context.DynamicPropertyRegistry -import org.springframework.test.context.DynamicPropertySource +import org.springframework.boot.testcontainers.service.connection.ServiceConnection +import org.testcontainers.junit.jupiter.Container +import org.testcontainers.junit.jupiter.Testcontainers import org.testcontainers.kafka.ConfluentKafkaContainer -import org.testcontainers.utility.DockerImageName -interface WithKafkaContainer { +@Testcontainers +@Suppress("UtilityClassWithPublicConstructor") +open class WithKafkaContainer { companion object { - private val kafkaImage: DockerImageName = - DockerImageName.parse("confluentinc/cp-kafka:7.6.0") - - private val kafkaContainer = ConfluentKafkaContainer(kafkaImage).apply { - withReuse(true) - } - + @Container + @ServiceConnection @JvmStatic - @DynamicPropertySource - fun properties(registry: DynamicPropertyRegistry) { - registry.add("spring.kafka.bootstrap-servers") { kafkaContainer.bootstrapServers } - } - - init { - kafkaContainer.start() + val kafkaContainer = ConfluentKafkaContainer("confluentinc/cp-kafka:7.6.0").apply { + withReuse(true) } } } diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/AggregatedTemporalQueryServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/AggregatedTemporalQueryServiceTests.kt index 775715ee5..3efe15704 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/AggregatedTemporalQueryServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/AggregatedTemporalQueryServiceTests.kt @@ -45,7 +45,7 @@ import java.util.UUID @SpringBootTest @ActiveProfiles("test") -class AggregatedTemporalQueryServiceTests : WithTimescaleContainer, WithKafkaContainer { +class AggregatedTemporalQueryServiceTests : WithTimescaleContainer, WithKafkaContainer() { @Autowired private lateinit var attributeInstanceService: AttributeInstanceService diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/AttributeInstanceServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/AttributeInstanceServiceTests.kt index a05fc17f5..5cca2a169 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/AttributeInstanceServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/AttributeInstanceServiceTests.kt @@ -39,10 +39,12 @@ import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_VALUE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DATE_TIME_TYPE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DEFAULT_VOCAB +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DELETED_AT_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_INSTANCE_ID_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_JSONPROPERTY_VALUE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_LANGUAGEPROPERTY_VALUE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_MODIFIED_AT_PROPERTY +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_NULL import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_OBSERVED_AT_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_PROPERTY_VALUE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_VOCABPROPERTY_VALUE @@ -77,14 +79,12 @@ import org.springframework.boot.test.context.SpringBootTest import org.springframework.data.r2dbc.core.R2dbcEntityTemplate import org.springframework.r2dbc.core.DatabaseClient import org.springframework.test.context.ActiveProfiles -import java.time.Instant -import java.time.ZoneOffset import java.time.ZonedDateTime import java.util.UUID @SpringBootTest @ActiveProfiles("test") -class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer { +class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer() { @Autowired private lateinit var attributeInstanceService: AttributeInstanceService @@ -101,7 +101,7 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer @Autowired private lateinit var r2dbcEntityTemplate: R2dbcEntityTemplate - private val now = Instant.now().atZone(ZoneOffset.UTC) + private val now = ngsiLdDateTime() private lateinit var incomingAttribute: Attribute private lateinit var outgoingAttribute: Attribute private lateinit var jsonAttribute: Attribute @@ -331,7 +331,7 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer entityAttributeService.create(attribute2) (1..10).forEach { _ -> - val observedAt = Instant.now().atZone(ZoneOffset.UTC) + val observedAt = ngsiLdDateTime() val attributeMetadata = AttributeMetadata( measuredValue = null, value = "some value", @@ -613,7 +613,7 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer } @Test - fun `it should create an attribute instance if it has a non null value`() = runTest { + fun `it should create an observed attribute instance if it has a non null value`() = runTest { val attributeInstanceService = spyk( AttributeInstanceService(databaseClient, searchProperties), recordPrivateCalls = true @@ -641,7 +641,7 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer ) ) - attributeInstanceService.addAttributeInstance( + attributeInstanceService.addObservedAttributeInstance( incomingAttribute.id, attributeMetadata, attributeValues @@ -675,7 +675,7 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer } @Test - fun `it should create an attribute instance with boolean value`() = runTest { + fun `it should create an observed attribute instance with boolean value`() = runTest { val attributeInstanceService = spyk( AttributeInstanceService(databaseClient, searchProperties), recordPrivateCalls = true @@ -703,7 +703,7 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer ) ) - attributeInstanceService.addAttributeInstance( + attributeInstanceService.addObservedAttributeInstance( incomingAttribute.id, attributeMetadata, attributeValues @@ -736,6 +736,61 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer } } + @Test + fun `it should create a deleted attribute instance`() = runTest { + val attributeInstanceService = spyk( + AttributeInstanceService(databaseClient, searchProperties), + recordPrivateCalls = true + ) + val deletedAt = ngsiLdDateTime() + val attributeValues = mapOf( + NGSILD_DELETED_AT_PROPERTY to listOf( + mapOf( + JSONLD_VALUE to deletedAt, + JSONLD_TYPE to NGSILD_DATE_TIME_TYPE + ) + ), + NGSILD_PROPERTY_VALUE to listOf( + mapOf( + JSONLD_VALUE to NGSILD_NULL + ) + ) + ) + + attributeInstanceService.addDeletedAttributeInstance( + incomingAttribute.id, + NGSILD_NULL, + deletedAt, + attributeValues + ) + + verify { + attributeInstanceService["create"]( + match { + it.time == deletedAt && + it.value == "urn:ngsi-ld:null" && + it.measuredValue == null && + it.payload.asString().matchContent( + """ + { + "https://uri.etsi.org/ngsi-ld/deletedAt":[{ + "@value":"$deletedAt", + "@type":"https://uri.etsi.org/ngsi-ld/DateTime" + }], + "https://uri.etsi.org/ngsi-ld/hasValue":[{ + "@value":"urn:ngsi-ld:null" + }], + "https://uri.etsi.org/ngsi-ld/instanceId":[{ + "@id":"${it.instanceId}" + }] + } + """.trimIndent() + ) + } + ) + } + } + @Test fun `it should modify attribute instance for a property`() = runTest { val attributeInstance = gimmeNumericPropertyAttributeInstance(incomingAttribute.id) diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/TemporalQueryServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/TemporalQueryServiceTests.kt index e04ae3310..21303c96b 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/TemporalQueryServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/TemporalQueryServiceTests.kt @@ -118,7 +118,7 @@ class TemporalQueryServiceTests { coEvery { entityQueryService.checkEntityExistence(any()) } returns Unit.right() coEvery { authorizationService.userCanReadEntity(any(), any()) } returns Unit.right() - coEvery { entityAttributeService.getForEntity(any(), any(), any()) } returns attributes + coEvery { entityAttributeService.getForEntity(any(), any(), any(), any()) } returns attributes coEvery { entityQueryService.retrieve(any()) } returns gimmeEntityPayload().right() coEvery { scopeService.retrieveHistory(any(), any()) } returns emptyList().right() coEvery { @@ -149,7 +149,7 @@ class TemporalQueryServiceTests { coVerify { entityQueryService.checkEntityExistence(entityUri) authorizationService.userCanReadEntity(entityUri, None) - entityAttributeService.getForEntity(entityUri, emptySet(), emptySet()) + entityAttributeService.getForEntity(entityUri, emptySet(), emptySet(), false) attributeInstanceService.search( match { temporalEntitiesQuery -> temporalEntitiesQuery.temporalQuery.timerel == TemporalQuery.Timerel.AFTER && @@ -246,11 +246,11 @@ class TemporalQueryServiceTests { ) coEvery { authorizationService.computeAccessRightFilter(any()) } returns { null } - coEvery { entityQueryService.queryEntities(any(), any<() -> String?>()) } returns listOf(entityUri) + coEvery { entityQueryService.queryEntities(any(), any(), any<() -> String?>()) } returns listOf(entityUri) coEvery { entityAttributeService.getForEntities(any(), any()) } returns listOf(attribute) - coEvery { entityQueryService.queryEntitiesCount(any(), any()) } returns 1.right() + coEvery { entityQueryService.queryEntitiesCount(any(), any(), any()) } returns 1.right() coEvery { scopeService.retrieveHistory(any(), any()) } returns emptyList().right() coEvery { entityQueryService.retrieve(any()) } returns gimmeEntityPayload().right() coEvery { @@ -305,6 +305,7 @@ class TemporalQueryServiceTests { paginationQuery = PaginationQuery(limit = 2, offset = 2), contexts = APIC_COMPOUND_CONTEXTS ), + false, any() ) scopeService.retrieveHistory(listOf(entityUri), any()) @@ -322,7 +323,7 @@ class TemporalQueryServiceTests { ) coEvery { authorizationService.computeAccessRightFilter(any()) } returns { null } - coEvery { entityQueryService.queryEntities(any(), any<() -> String?>()) } returns listOf(entityUri) + coEvery { entityQueryService.queryEntities(any(), any(), any<() -> String?>()) } returns listOf(entityUri) coEvery { entityAttributeService.getForEntities(any(), any()) } returns listOf(attribute) @@ -331,7 +332,7 @@ class TemporalQueryServiceTests { attributeInstanceService.search(any(), any>()) } returns emptyList().right() coEvery { entityQueryService.retrieve(any()) } returns gimmeEntityPayload().right() - coEvery { entityQueryService.queryEntitiesCount(any(), any()) } returns 1.right() + coEvery { entityQueryService.queryEntitiesCount(any(), any(), any()) } returns 1.right() temporalQueryService.queryTemporalEntities( TemporalEntitiesQueryFromGet( diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/TemporalServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/TemporalServiceTests.kt new file mode 100644 index 000000000..9e04afa27 --- /dev/null +++ b/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/TemporalServiceTests.kt @@ -0,0 +1,127 @@ +package com.egm.stellio.search.temporal.service + +import arrow.core.left +import arrow.core.right +import com.egm.stellio.search.authorization.service.AuthorizationService +import com.egm.stellio.search.entity.service.EntityQueryService +import com.egm.stellio.search.entity.service.EntityService +import com.egm.stellio.shared.model.ResourceNotFoundException +import com.egm.stellio.shared.util.INCOMING_PROPERTY +import com.egm.stellio.shared.util.loadAndExpandSampleData +import com.egm.stellio.shared.util.shouldSucceed +import com.egm.stellio.shared.util.toUri +import com.ninjasquad.springmockk.MockkBean +import io.mockk.coEvery +import io.mockk.coVerify +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.test.context.ActiveProfiles + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, classes = [TemporalService::class]) +@ActiveProfiles("test") +@ExperimentalCoroutinesApi +class TemporalServiceTests { + + @Autowired + private lateinit var temporalService: TemporalService + + @MockkBean + private lateinit var entityQueryService: EntityQueryService + + @MockkBean + private lateinit var entityService: EntityService + + @MockkBean + private lateinit var attributeInstanceService: AttributeInstanceService + + @MockkBean + private lateinit var authorizationService: AuthorizationService + + private val entityUri = "urn:ngsi-ld:BeeHive:TESTC".toUri() + private val sub = "0123456789-1234-5678-987654321" + + private fun mockkAuthorizationForCreation() { + coEvery { authorizationService.userCanCreateEntities(any()) } returns Unit.right() + coEvery { authorizationService.userCanUpdateEntity(any(), any()) } returns Unit.right() + coEvery { authorizationService.createOwnerRight(any(), any()) } returns Unit.right() + } + + @Test + fun `it should ask to create a temporal entity if it does not exist yet`() = runTest { + mockkAuthorizationForCreation() + coEvery { + entityQueryService.isMarkedAsDeleted(entityUri) + } returns ResourceNotFoundException("Entity does not exist").left() + coEvery { entityService.createEntity(any(), any(), any()) } returns Unit.right() + coEvery { entityService.upsertAttributes(any(), any(), any()) } returns Unit.right() + + val expandedEntity = loadAndExpandSampleData("temporal/beehive_create_temporal_entity.jsonld") + + temporalService.createOrUpdateTemporalEntity(entityUri, expandedEntity, sub).shouldSucceed() + } + + @Test + fun `it should ask to create a temporal entity if it already exists but is deleted`() = runTest { + mockkAuthorizationForCreation() + coEvery { entityQueryService.isMarkedAsDeleted(entityUri) } returns false.right() + coEvery { entityService.createEntity(any(), any(), any()) } returns Unit.right() + coEvery { entityService.upsertAttributes(any(), any(), any()) } returns Unit.right() + + val expandedEntity = loadAndExpandSampleData("temporal/beehive_create_temporal_entity.jsonld") + + temporalService.createOrUpdateTemporalEntity(entityUri, expandedEntity, sub).shouldSucceed() + } + + @Test + fun `it should ask to upsert a temporal entity if it already exists but is not deleted`() = runTest { + mockkAuthorizationForCreation() + coEvery { entityQueryService.isMarkedAsDeleted(entityUri) } returns false.right() + coEvery { entityService.upsertAttributes(any(), any(), any()) } returns Unit.right() + + val expandedEntity = loadAndExpandSampleData("temporal/beehive_create_temporal_entity.jsonld") + + temporalService.createOrUpdateTemporalEntity(entityUri, expandedEntity, sub).shouldSucceed() + } + + @Test + fun `it should ask to permanently delete a temporal entity`() = runTest { + coEvery { entityService.permanentlyDeleteEntity(any(), any()) } returns Unit.right() + + temporalService.deleteEntity(entityUri, sub).shouldSucceed() + + coVerify(exactly = 1) { + entityService.permanentlyDeleteEntity(entityUri, sub) + } + } + + @Test + fun `it should ask to permanently delete a temporal attribute`() = runTest { + coEvery { entityService.permanentlyDeleteAttribute(any(), any(), any(), any(), any()) } returns Unit.right() + + temporalService.deleteAttribute(entityUri, INCOMING_PROPERTY, null).shouldSucceed() + + coVerify(exactly = 1) { + entityService.permanentlyDeleteAttribute(entityUri, INCOMING_PROPERTY, null) + } + } + + @Test + fun `it should ask to permanently delete a temporal attribute instance`() = runTest { + val instanceId = "urn:ngsi-ld:Instance:01".toUri() + + coEvery { entityQueryService.checkEntityExistence(entityUri) } returns Unit.right() + coEvery { authorizationService.userCanUpdateEntity(entityUri, any()) } returns Unit.right() + coEvery { + attributeInstanceService.deleteInstance(entityUri, INCOMING_PROPERTY, instanceId) + } returns Unit.right() + + temporalService.deleteAttributeInstance(entityUri, INCOMING_PROPERTY, instanceId).shouldSucceed() + + coVerify(exactly = 1) { + attributeInstanceService.deleteInstance(entityUri, INCOMING_PROPERTY, instanceId) + } + } +} diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/temporal/web/TemporalEntityHandlerTestCommon.kt b/search-service/src/test/kotlin/com/egm/stellio/search/temporal/web/TemporalEntityHandlerTestCommon.kt index 0102fd19a..f2462beff 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/temporal/web/TemporalEntityHandlerTestCommon.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/temporal/web/TemporalEntityHandlerTestCommon.kt @@ -1,7 +1,6 @@ package com.egm.stellio.search.temporal.web import com.egm.stellio.search.common.config.SearchProperties -import com.egm.stellio.search.entity.service.EntityService import com.egm.stellio.search.temporal.service.TemporalQueryService import com.egm.stellio.search.temporal.service.TemporalService import com.egm.stellio.shared.config.ApplicationProperties @@ -30,9 +29,6 @@ open class TemporalEntityHandlerTestCommon { @MockkBean(relaxed = true) protected lateinit var temporalQueryService: TemporalQueryService - @MockkBean - protected lateinit var entityService: EntityService - @BeforeAll fun configureWebClientDefaults() { webClient = webClient.mutate() diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/temporal/web/TemporalEntityHandlerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/temporal/web/TemporalEntityHandlerTests.kt index 24b3a4126..73d2f7541 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/temporal/web/TemporalEntityHandlerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/temporal/web/TemporalEntityHandlerTests.kt @@ -972,7 +972,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { @Test fun `delete temporal entity should return a 204 if an entity has been successfully deleted`() { - coEvery { entityService.deleteEntity(any(), any()) } returns Unit.right() + coEvery { temporalService.deleteEntity(any(), any()) } returns Unit.right() webClient.delete() .uri("/ngsi-ld/v1/temporal/entities/$entityUri") @@ -981,14 +981,14 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { .expectBody().isEmpty coVerify { - entityService.deleteEntity(eq(entityUri), eq(sub.value)) + temporalService.deleteEntity(eq(entityUri), eq(sub.value)) } } @Test fun `delete temporal entity should return a 404 if entity to be deleted has not been found`() { coEvery { - entityService.deleteEntity(entityUri, sub.getOrNull()) + temporalService.deleteEntity(entityUri, sub.getOrNull()) } returns ResourceNotFoundException(entityNotFoundMessage(entityUri.toString())).left() webClient.delete() @@ -1034,7 +1034,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { @Test fun `delete temporal entity should return a 500 if entity could not be deleted`() { coEvery { - entityService.deleteEntity(any(), any()) + temporalService.deleteEntity(any(), any()) } throws RuntimeException("Unexpected server error") webClient.delete() @@ -1055,7 +1055,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { @Test fun `delete temporal entity should return a 403 is user is not authorized to delete an entity`() { coEvery { - entityService.deleteEntity(any(), any()) + temporalService.deleteEntity(any(), any()) } returns AccessDeniedException("User forbidden admin access to entity $entityUri").left() webClient.delete() @@ -1077,7 +1077,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { @Test fun `delete attribute temporal should return a 204 if the attribute has been successfully deleted`() { coEvery { - entityService.deleteAttribute(any(), any(), any(), any(), any()) + temporalService.deleteAttribute(any(), any(), any(), any(), any()) } returns Unit.right() webClient.method(HttpMethod.DELETE) @@ -1089,7 +1089,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { .expectBody().isEmpty coVerify { - entityService.deleteAttribute( + temporalService.deleteAttribute( eq(entityUri), eq(TEMPERATURE_PROPERTY), null, @@ -1102,7 +1102,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { @Test fun `delete attribute temporal should delete all instances if deleteAll flag is true`() { coEvery { - entityService.deleteAttribute(any(), any(), any(), any(), any()) + temporalService.deleteAttribute(any(), any(), any(), any(), any()) } returns Unit.right() webClient.method(HttpMethod.DELETE) @@ -1114,7 +1114,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { .expectBody().isEmpty coVerify { - entityService.deleteAttribute( + temporalService.deleteAttribute( eq(entityUri), eq(TEMPERATURE_PROPERTY), null, @@ -1128,7 +1128,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { fun `delete attribute temporal should delete instance with the provided datasetId`() { val datasetId = "urn:ngsi-ld:Dataset:temperature:1" coEvery { - entityService.deleteAttribute(any(), any(), any(), any(), any()) + temporalService.deleteAttribute(any(), any(), any(), any(), any()) } returns Unit.right() webClient.method(HttpMethod.DELETE) @@ -1140,7 +1140,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { .expectBody().isEmpty coVerify { - entityService.deleteAttribute( + temporalService.deleteAttribute( eq(entityUri), eq(TEMPERATURE_PROPERTY), eq(datasetId.toUri()), @@ -1153,7 +1153,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { @Test fun `delete attribute temporal should return a 404 if the entity is not found`() { coEvery { - entityService.deleteAttribute(any(), any(), any(), any(), any()) + temporalService.deleteAttribute(any(), any(), any(), any(), any()) } returns ResourceNotFoundException(entityNotFoundMessage(entityUri.toString())).left() webClient.method(HttpMethod.DELETE) @@ -1176,7 +1176,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { @Test fun `delete attribute temporal should return a 404 if the attribute is not found`() { coEvery { - entityService.deleteAttribute(any(), any(), any(), any(), any()) + temporalService.deleteAttribute(any(), any(), any(), any(), any()) } returns ResourceNotFoundException("Attribute Not Found").left() webClient.method(HttpMethod.DELETE) @@ -1199,7 +1199,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { @Test fun `delete attribute temporal should return a 400 if the request is not correct`() { coEvery { - entityService.deleteAttribute(any(), any(), any(), any(), any()) + temporalService.deleteAttribute(any(), any(), any(), any(), any()) } returns BadRequestDataException("Something is wrong with the request").left() webClient.method(HttpMethod.DELETE) @@ -1273,7 +1273,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { @Test fun `delete attribute temporal should return a 403 if user is not allowed to update entity`() { coEvery { - entityService.deleteAttribute(any(), any(), any(), any(), any()) + temporalService.deleteAttribute(any(), any(), any(), any(), any()) } returns AccessDeniedException("User forbidden write access to entity $entityUri").left() webClient.method(HttpMethod.DELETE) diff --git a/search-service/src/test/resources/ngsild/fragments/beehive_mergeAttribute_null.json b/search-service/src/test/resources/ngsild/fragments/beehive_mergeAttribute_null.json new file mode 100644 index 000000000..0f3603a67 --- /dev/null +++ b/search-service/src/test/resources/ngsild/fragments/beehive_mergeAttribute_null.json @@ -0,0 +1,6 @@ +{ + "incoming": { + "type": "Property", + "value": "urn:ngsi-ld:null" + } +} diff --git a/search-service/src/test/resources/ngsild/fragments/beehive_mergeAttribute_null_fragment.json b/search-service/src/test/resources/ngsild/fragments/beehive_mergeAttribute_null_fragment.json new file mode 100644 index 000000000..acf095c32 --- /dev/null +++ b/search-service/src/test/resources/ngsild/fragments/beehive_mergeAttribute_null_fragment.json @@ -0,0 +1,4 @@ +{ + "type": "Property", + "value": "urn:ngsi-ld:null" +} diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/model/ExpandedMembers.kt b/shared/src/main/kotlin/com/egm/stellio/shared/model/ExpandedMembers.kt index b4ed57866..7e53f6226 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/model/ExpandedMembers.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/model/ExpandedMembers.kt @@ -197,6 +197,9 @@ fun ExpandedAttributeInstance.getRelationshipObject(name: String): Either)?.get(JSONLD_ID)?.toUri() +fun ExpandedAttributeInstance.getRelationshipId(): URI? = + (this[NGSILD_RELATIONSHIP_OBJECT]?.get(0) as? Map)?.get(JSONLD_ID)?.toUri() + fun ExpandedAttributeInstance.getScopes(): List? = when (val rawScopes = this.getMemberValue(NGSILD_SCOPE_PROPERTY)) { is String -> listOf(rawScopes) diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/queryparameter/QueryParameter.kt b/shared/src/main/kotlin/com/egm/stellio/shared/queryparameter/QueryParameter.kt index 00d91392b..3762001e6 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/queryparameter/QueryParameter.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/queryparameter/QueryParameter.kt @@ -35,6 +35,9 @@ enum class QueryParameter( LASTN("lastN"), TIMEPROPERTY("timeproperty"), + // authz + INCLUDE_DELETED("includeDeleted"), + // pagination COUNT("count"), OFFSET("offset"), diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt index e05b2391d..8fe916f65 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt @@ -56,7 +56,6 @@ const val INCONSISTENT_VALUES_IN_AGGREGATION_MESSAGE = const val ENTITIY_CREATION_FORBIDDEN_MESSAGE = "User forbidden to create entity" const val ENTITIY_READ_FORBIDDEN_MESSAGE = "User forbidden to read entity" const val ENTITY_UPDATE_FORBIDDEN_MESSAGE = "User forbidden to modify entity" -const val ENTITY_DELETE_FORBIDDEN_MESSAGE = "User forbidden to delete entity" const val ENTITY_ADMIN_FORBIDDEN_MESSAGE = "User forbidden to admin entity" const val ENTITY_REMOVE_OWNERSHIP_FORBIDDEN_MESSAGE = "User forbidden to remove ownership of entity" const val ENTITY_ALREADY_EXISTS_MESSAGE = "Entity already exists" diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/AuthUtils.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/AuthUtils.kt index 0d10c7f76..9bbf3ed68 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/AuthUtils.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/AuthUtils.kt @@ -67,6 +67,8 @@ object AuthContextModel { const val AUTH_PROP_SAP = AUTHORIZATION_ONTOLOGY + AUTH_TERM_SAP const val AUTH_TERM_RIGHT = "right" const val AUTH_PROP_RIGHT: ExpandedTerm = AUTHORIZATION_ONTOLOGY + AUTH_TERM_RIGHT + const val AUTH_TERM_IS_DELETED = "isDeleted" + const val AUTH_PROP_IS_DELETED: ExpandedTerm = AUTHORIZATION_ONTOLOGY + AUTH_TERM_IS_DELETED const val AUTH_TERM_IS_MEMBER_OF = "isMemberOf" const val AUTH_REL_IS_MEMBER_OF: ExpandedTerm = AUTHORIZATION_ONTOLOGY + AUTH_TERM_IS_MEMBER_OF diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/DataRepresentationUtils.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/DataRepresentationUtils.kt index 0ec2645aa..5aa04074e 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/DataRepresentationUtils.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/DataRepresentationUtils.kt @@ -40,9 +40,9 @@ fun String.checkNameIsNgsiLdSupported(): Either = * Returns whether the given string is a supported name as defined in 4.6.2 */ private fun String.isNgsiLdSupportedName(): Boolean = - this.all { char -> char.isLetterOrDigit() || listOf(':', '_').contains(char) } + this.all { char -> char.isLetterOrDigit() || listOf(':', '_', '@').contains(char) } -val scopeNameRegex: Regex = """^/?\p{L}+[\p{L}\w_]*(/\p{L}+[\p{L}\w_]*)*$""".toRegex() +val scopeNameRegex: Regex = """^/?\p{L}+[\p{L}\w]*(/\p{L}+[\p{L}\w]*)*$""".toRegex() fun Any.checkScopesNamesAreNgsiLdSupported(): Either { val rawScope = this diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt index c07320615..ffaeb7ad3 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt @@ -94,6 +94,8 @@ object JsonLdUtils { const val NGSILD_NONE_TERM = "@none" const val NGSILD_DATASET_TERM = "dataset" const val NGSILD_ENTITY_TERM = "entity" + const val NGSILD_NULL = "urn:ngsi-ld:null" + val JSONLD_EXPANDED_ENTITY_SPECIFIC_MEMBERS = setOf(JSONLD_TYPE, NGSILD_SCOPE_PROPERTY) // List of members that are part of a core entity base definition (i.e., without attributes) @@ -124,6 +126,8 @@ object JsonLdUtils { val NGSILD_SYSATTRS_PROPERTIES = setOf(NGSILD_CREATED_AT_PROPERTY, NGSILD_MODIFIED_AT_PROPERTY) const val NGSILD_OBSERVED_AT_TERM = "observedAt" const val NGSILD_OBSERVED_AT_PROPERTY = "https://uri.etsi.org/ngsi-ld/$NGSILD_OBSERVED_AT_TERM" + const val NGSILD_DELETED_AT_TERM = "deletedAt" + const val NGSILD_DELETED_AT_PROPERTY = "https://uri.etsi.org/ngsi-ld/$NGSILD_DELETED_AT_TERM" const val NGSILD_UNIT_CODE_PROPERTY = "https://uri.etsi.org/ngsi-ld/unitCode" const val NGSILD_UNIT_CODE_TERM = "unitCode" const val NGSILD_LOCATION_TERM = "location" @@ -295,13 +299,13 @@ object JsonLdUtils { if (NGSILD_GEO_PROPERTIES_TERMS.contains(it.key)) { when (it.value) { is Map<*, *> -> { - val geoProperty = it.value as MutableMap - val wktGeometry = throwingGeoJsonToWkt(geoProperty[JSONLD_VALUE_TERM]!! as Map) + val geoProperty = it.value as Map + val wktGeometry = geoPropertyToWKTOrNull(geoProperty[JSONLD_VALUE_TERM]!!) geoProperty.plus(JSONLD_VALUE_TERM to wktGeometry) } is List<*> -> { (it.value as List>).map { geoProperty -> - val wktGeometry = throwingGeoJsonToWkt(geoProperty[JSONLD_VALUE_TERM] as Map) + val wktGeometry = geoPropertyToWKTOrNull(geoProperty[JSONLD_VALUE_TERM]!!) geoProperty.plus(JSONLD_VALUE_TERM to wktGeometry) } } @@ -310,6 +314,12 @@ object JsonLdUtils { } else it.value } + private fun geoPropertyToWKTOrNull(geoPropertyValue: Any): String = + if (geoPropertyValue is String && geoPropertyValue == NGSILD_NULL) + NGSILD_NULL + else + throwingGeoJsonToWkt(geoPropertyValue as Map) + private fun restoreGeoPropertyFromWKT(): (Map.Entry) -> Any = { if (NGSILD_GEO_PROPERTIES_TERMS.contains(it.key)) { when (it.value) { diff --git a/shared/src/testFixtures/resources/jsonld-contexts/authorization.jsonld b/shared/src/testFixtures/resources/jsonld-contexts/authorization.jsonld index 28361e902..f086636cc 100644 --- a/shared/src/testFixtures/resources/jsonld-contexts/authorization.jsonld +++ b/shared/src/testFixtures/resources/jsonld-contexts/authorization.jsonld @@ -9,6 +9,7 @@ "clientId": "https://ontology.eglobalmark.com/authorization#clientId", "familyName": "https://ontology.eglobalmark.com/authorization#familyName", "givenName": "https://ontology.eglobalmark.com/authorization#givenName", + "isDeleted": "https://ontology.eglobalmark.com/authorization#isDeleted", "isMemberOf": "https://ontology.eglobalmark.com/authorization#isMemberOf", "isOwner": "https://ontology.eglobalmark.com/authorization#isOwner", "kind": "https://ontology.eglobalmark.com/authorization#kind", diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/SubscriptionServiceTests.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/SubscriptionServiceTests.kt index 2c2ca9709..fc33b82dd 100644 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/SubscriptionServiceTests.kt +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/SubscriptionServiceTests.kt @@ -76,7 +76,7 @@ import kotlin.time.Duration @SpringBootTest @ActiveProfiles("test") @TestPropertySource(properties = ["application.authentication.enabled=false"]) -class SubscriptionServiceTests : WithTimescaleContainer, WithKafkaContainer { +class SubscriptionServiceTests : WithTimescaleContainer, WithKafkaContainer() { @Autowired private lateinit var subscriptionService: SubscriptionService diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithKafkaContainer.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithKafkaContainer.kt index cbe761176..2b40e471c 100644 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithKafkaContainer.kt +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithKafkaContainer.kt @@ -1,29 +1,21 @@ package com.egm.stellio.subscription.support -import org.springframework.test.context.DynamicPropertyRegistry -import org.springframework.test.context.DynamicPropertySource +import org.springframework.boot.testcontainers.service.connection.ServiceConnection +import org.testcontainers.junit.jupiter.Container +import org.testcontainers.junit.jupiter.Testcontainers import org.testcontainers.kafka.ConfluentKafkaContainer -import org.testcontainers.utility.DockerImageName -interface WithKafkaContainer { +@Testcontainers +@Suppress("UtilityClassWithPublicConstructor") +open class WithKafkaContainer { companion object { - private val kafkaImage: DockerImageName = - DockerImageName.parse("confluentinc/cp-kafka:7.6.0") - - private val kafkaContainer = ConfluentKafkaContainer(kafkaImage).apply { - withReuse(true) - } - + @Container + @ServiceConnection @JvmStatic - @DynamicPropertySource - fun properties(registry: DynamicPropertyRegistry) { - registry.add("spring.kafka.bootstrap-servers") { kafkaContainer.bootstrapServers } - } - - init { - kafkaContainer.start() + val kafkaContainer = ConfluentKafkaContainer("confluentinc/cp-kafka:7.6.0").apply { + withReuse(true) } } }