diff --git a/build.gradle.kts b/build.gradle.kts index 5b1074a5e..4436971f4 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -79,6 +79,7 @@ subprojects { runtimeOnly("io.micrometer:micrometer-registry-prometheus") testImplementation("org.springframework.boot:spring-boot-starter-test") + testImplementation("org.springframework.boot:spring-boot-testcontainers") testImplementation("io.projectreactor:reactor-test") testImplementation("com.ninja-squad:springmockk:4.0.2") testImplementation("org.springframework.security:spring-security-test") diff --git a/search-service/config/detekt/baseline.xml b/search-service/config/detekt/baseline.xml index 7f380d5b4..f467d3c62 100644 --- a/search-service/config/detekt/baseline.xml +++ b/search-service/config/detekt/baseline.xml @@ -5,11 +5,11 @@ ClassNaming:V0_29_JsonLd_migrationTests.kt$V0_29_JsonLd_migrationTests ClassNaming:V0_29__JsonLd_migration.kt$V0_29__JsonLd_migration : BaseJavaMigration ComplexCondition:EntitiesQueryUtils.kt$geoQuery == null && q.isNullOrEmpty() && typeSelection.isNullOrEmpty() && attrs.isEmpty() - ComplexCondition:EntityQueryService.kt$EntityQueryService$it && !inverse || !it && inverse Filename:V0_29__JsonLd_migration.kt$db.migration.V0_29__JsonLd_migration.kt LongMethod:AttributeInstanceService.kt$AttributeInstanceService$@Transactional suspend fun create(attributeInstance: AttributeInstance): Either<APIException, Unit> LongMethod:EnabledAuthorizationServiceTests.kt$EnabledAuthorizationServiceTests$@Test fun `it should return serialized access control entities with other rigths if user is owner`() LongMethod:EntityAccessControlHandler.kt$EntityAccessControlHandler$@PostMapping("/{subjectId}/attrs", consumes = [MediaType.APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE]) suspend fun addRightsOnEntities( @RequestHeader httpHeaders: HttpHeaders, @PathVariable subjectId: String, @RequestBody requestBody: Mono<String>, @AllowedParameters @RequestParam queryParams: MultiValueMap<String, String> ): ResponseEntity<*> + LongMethod:EntityEventService.kt$EntityEventService$private fun publishAttributeChangeEvent( sub: String?, tenantName: String, entityId: URI, entityTypesAndPayload: Pair<List<ExpandedTerm>, String>, attributeOperationResult: SucceededAttributeOperationResult ) LongMethod:EntityHandler.kt$EntityHandler$@GetMapping("/{entityId}", produces = [APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE, GEO_JSON_CONTENT_TYPE]) suspend fun getByURI( @RequestHeader httpHeaders: HttpHeaders, @PathVariable entityId: URI, @AllowedParameters( implemented = [ QP.OPTIONS, QP.TYPE, QP.ATTRS, QP.GEOMETRY_PROPERTY, QP.LANG, QP.CONTAINED_BY, QP.JOIN, QP.JOIN_LEVEL, QP.DATASET_ID, ], notImplemented = [QP.FORMAT, QP.PICK, QP.OMIT, QP.ENTITY_MAP, QP.LOCAL, QP.VIA] ) @RequestParam queryParams: MultiValueMap<String, String> ): ResponseEntity<*> LongMethod:EntityHandler.kt$EntityHandler$@GetMapping(produces = [APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE, GEO_JSON_CONTENT_TYPE]) suspend fun getEntities( @RequestHeader httpHeaders: HttpHeaders, @AllowedParameters( implemented = [ QP.OPTIONS, QP.COUNT, QP.OFFSET, QP.LIMIT, QP.ID, QP.TYPE, QP.ID_PATTERN, QP.ATTRS, QP.Q, QP.GEOMETRY, QP.GEOREL, QP.COORDINATES, QP.GEOPROPERTY, QP.GEOMETRY_PROPERTY, QP.LANG, QP.SCOPEQ, QP.CONTAINED_BY, QP.JOIN, QP.JOIN_LEVEL, QP.DATASET_ID, ], notImplemented = [QP.FORMAT, QP.PICK, QP.OMIT, QP.EXPAND_VALUES, QP.CSF, QP.ENTITY_MAP, QP.LOCAL, QP.VIA] ) @RequestParam queryParams: MultiValueMap<String, String> ): ResponseEntity<*> LongMethod:LinkedEntityServiceTests.kt$LinkedEntityServiceTests$@Test fun `it should inline entities up to the asked 2nd level`() @@ -28,7 +28,6 @@ LongParameterList:EntityAttributeService.kt$EntityAttributeService$( entityId: URI, attributeName: ExpandedTerm, attributeMetadata: AttributeMetadata, createdAt: ZonedDateTime, attributePayload: ExpandedAttributeInstance, sub: Sub? ) LongParameterList:EntityAttributeService.kt$EntityAttributeService$( entityUri: URI, ngsiLdAttributes: List<NgsiLdAttribute>, expandedAttributes: ExpandedAttributes, createdAt: ZonedDateTime, observedAt: ZonedDateTime?, sub: Sub? ) LongParameterList:EntityAttributeService.kt$EntityAttributeService$( entityUri: URI, ngsiLdAttributes: List<NgsiLdAttribute>, expandedAttributes: ExpandedAttributes, disallowOverwrite: Boolean, createdAt: ZonedDateTime, sub: Sub? ) - LongParameterList:EntityEventService.kt$EntityEventService$( updatedDetails: UpdatedDetails, sub: String?, tenantName: String, entityId: URI, entityTypesAndPayload: Pair<List<ExpandedTerm>, String>, serializedAttribute: Pair<ExpandedTerm, String>, overwrite: Boolean ) LongParameterList:TemporalEntityHandler.kt$TemporalEntityHandler$( @RequestHeader httpHeaders: HttpHeaders, @PathVariable entityId: URI, @PathVariable attrId: String, @PathVariable instanceId: URI, @RequestBody requestBody: Mono<String>, @AllowedParameters(notImplemented = [QP.LOCAL, QP.VIA]) @RequestParam queryParams: MultiValueMap<String, String> ) LongParameterList:V0_29__JsonLd_migration.kt$V0_29__JsonLd_migration$( entityId: URI, attributeName: ExpandedTerm, datasetId: URI?, attributePayload: ExpandedAttributeInstance, ngsiLdAttributeInstance: NgsiLdAttributeInstance, defaultCreatedAt: ZonedDateTime ) NestedBlockDepth:V0_29__JsonLd_migration.kt$V0_29__JsonLd_migration$override fun migrate(context: Context) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/listener/IAMListener.kt b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/listener/IAMListener.kt index d4fa226e6..a7e32bec2 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/listener/IAMListener.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/listener/IAMListener.kt @@ -125,7 +125,7 @@ class IAMListener( // (if it no longer exists, it fails because of access rights checks) if (searchProperties.onOwnerDeleteCascadeEntities && subjectType == SubjectType.USER) { entityAccessRightsService.getEntitiesIdsOwnedBySubject(sub).getOrNull()?.forEach { entityId -> - entityService.deleteEntity(entityId, sub) + entityService.permanentlyDeleteEntity(entityId, sub) } Unit.right() } else Unit.right() diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/model/EntityAccessRights.kt b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/model/EntityAccessRights.kt index 022110962..7528f783c 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/model/EntityAccessRights.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/model/EntityAccessRights.kt @@ -6,6 +6,7 @@ import com.egm.stellio.shared.model.addNonReifiedProperty import com.egm.stellio.shared.model.addSubAttribute import com.egm.stellio.shared.util.AccessRight import com.egm.stellio.shared.util.AuthContextModel +import com.egm.stellio.shared.util.AuthContextModel.AUTH_PROP_IS_DELETED import com.egm.stellio.shared.util.AuthContextModel.AUTH_PROP_RIGHT import com.egm.stellio.shared.util.AuthContextModel.AUTH_PROP_SAP import com.egm.stellio.shared.util.AuthContextModel.AUTH_PROP_SUBJECT_INFO @@ -27,6 +28,7 @@ import java.net.URI data class EntityAccessRights( val id: URI, val types: List, + val isDeleted: Boolean = false, // right the current user has on the entity val right: AccessRight, val specificAccessPolicy: AuthContextModel.SpecificAccessPolicy? = null, @@ -55,6 +57,8 @@ data class EntityAccessRights( resultEntity[JSONLD_ID] = id.toString() resultEntity[JSONLD_TYPE] = types + if (isDeleted) + resultEntity[AUTH_PROP_IS_DELETED] = buildExpandedPropertyValue(true) resultEntity[AUTH_PROP_RIGHT] = buildExpandedPropertyValue(right.attributeName) specificAccessPolicy?.run { diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/AuthorizationService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/AuthorizationService.kt index c0dd9d0fe..c79064cfa 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/AuthorizationService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/AuthorizationService.kt @@ -23,6 +23,7 @@ interface AuthorizationService { suspend fun getAuthorizedEntities( entitiesQuery: EntitiesQueryFromGet, + includeDeleted: Boolean, contexts: List, sub: Option ): Either>> diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/DisabledAuthorizationService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/DisabledAuthorizationService.kt index f27debb04..5b56b8a42 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/DisabledAuthorizationService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/DisabledAuthorizationService.kt @@ -41,6 +41,7 @@ class DisabledAuthorizationService : AuthorizationService { override suspend fun getAuthorizedEntities( entitiesQuery: EntitiesQueryFromGet, + includeDeleted: Boolean, contexts: List, sub: Option ): Either>> = Pair(-1, emptyList()).right() diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/EnabledAuthorizationService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/EnabledAuthorizationService.kt index 53e3e31cf..65a64a3cf 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/EnabledAuthorizationService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/EnabledAuthorizationService.kt @@ -108,6 +108,7 @@ class EnabledAuthorizationService( override suspend fun getAuthorizedEntities( entitiesQuery: EntitiesQueryFromGet, + includeDeleted: Boolean, contexts: List, sub: Option ): Either>> = either { @@ -115,9 +116,8 @@ class EnabledAuthorizationService( val entitiesAccessRights = entityAccessRightsService.getSubjectAccessRights( sub, accessRights, - entitiesQuery.typeSelection, - entitiesQuery.ids, - entitiesQuery.paginationQuery + entitiesQuery, + includeDeleted ).bind() // for each entity user is admin or creator of, retrieve the full details of rights other users have on it @@ -148,7 +148,8 @@ class EnabledAuthorizationService( sub, accessRights, entitiesQuery.typeSelection, - entitiesQuery.ids + entitiesQuery.ids, + includeDeleted ).bind() Pair(count, entitiesAccessControlWithSubjectRights) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/EntityAccessRightsService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/EntityAccessRightsService.kt index e85454db8..fb6b3ce12 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/EntityAccessRightsService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/service/EntityAccessRightsService.kt @@ -18,13 +18,13 @@ import com.egm.stellio.search.common.util.toJsonString import com.egm.stellio.search.common.util.toList import com.egm.stellio.search.common.util.toOptionalEnum import com.egm.stellio.search.common.util.toUri +import com.egm.stellio.search.entity.model.EntitiesQueryFromGet import com.egm.stellio.shared.config.ApplicationProperties import com.egm.stellio.shared.model.APIException import com.egm.stellio.shared.model.AccessDeniedException import com.egm.stellio.shared.model.EntityTypeSelection import com.egm.stellio.shared.model.NgsiLdAttribute import com.egm.stellio.shared.model.ResourceNotFoundException -import com.egm.stellio.shared.queryparameter.PaginationQuery import com.egm.stellio.shared.util.AccessRight import com.egm.stellio.shared.util.AccessRight.CAN_ADMIN import com.egm.stellio.shared.util.AccessRight.CAN_READ @@ -220,30 +220,32 @@ class EntityAccessRightsService( suspend fun getSubjectAccessRights( sub: Option, accessRights: List, - type: EntityTypeSelection? = null, - ids: Set? = null, - paginationQuery: PaginationQuery, + entitiesQuery: EntitiesQueryFromGet, + includeDeleted: Boolean = false ): Either> = either { + val ids = entitiesQuery.ids + val typeSelection = entitiesQuery.typeSelection val subjectUuids = subjectReferentialService.getSubjectAndGroupsUUID(sub).bind() val isStellioAdmin = subjectReferentialService.hasStellioAdminRole(subjectUuids).bind() databaseClient .sql( """ - SELECT ep.entity_id, ep.types, ear.access_right, ep.specific_access_policy + SELECT ep.entity_id, ep.types, ear.access_right, ep.specific_access_policy, ep.deleted_at FROM entity_access_rights ear LEFT JOIN entity_payload ep ON ear.entity_id = ep.entity_id WHERE ${if (isStellioAdmin) "1 = 1" else "subject_id IN (:subject_uuids)" } ${if (accessRights.isNotEmpty()) " AND access_right IN (:access_rights)" else ""} - ${if (!type.isNullOrEmpty()) " AND (${buildTypeQuery(type)})" else ""} - ${if (!ids.isNullOrEmpty()) " AND ear.entity_id IN (:entities_ids)" else ""} + ${if (!typeSelection.isNullOrEmpty()) " AND (${buildTypeQuery(typeSelection)})" else ""} + ${if (ids.isNotEmpty()) " AND ear.entity_id IN (:entities_ids)" else ""} + ${if (!includeDeleted) " AND deleted_at IS NULL" else ""} ORDER BY entity_id LIMIT :limit OFFSET :offset; """.trimIndent() ) - .bind("limit", paginationQuery.limit) - .bind("offset", paginationQuery.offset) + .bind("limit", entitiesQuery.paginationQuery.limit) + .bind("offset", entitiesQuery.paginationQuery.offset) .let { if (!isStellioAdmin) it.bind("subject_uuids", subjectUuids) @@ -255,7 +257,7 @@ class EntityAccessRightsService( else it } .let { - if (!ids.isNullOrEmpty()) + if (ids.isNotEmpty()) it.bind("entities_ids", ids) else it } @@ -268,6 +270,7 @@ class EntityAccessRightsService( EntityAccessRights( ear.id, ear.types, + ear.isDeleted, entityAccessRights.maxOf { it.right }, ear.specificAccessPolicy ) @@ -278,7 +281,8 @@ class EntityAccessRightsService( sub: Option, accessRights: List, type: EntityTypeSelection? = null, - ids: Set? = null + ids: Set? = null, + includeDeleted: Boolean = false ): Either = either { val subjectUuids = subjectReferentialService.getSubjectAndGroupsUUID(sub).bind() val isStellioAdmin = subjectReferentialService.hasStellioAdminRole(subjectUuids).bind() @@ -293,6 +297,7 @@ class EntityAccessRightsService( ${if (accessRights.isNotEmpty()) " AND access_right IN (:access_rights)" else ""} ${if (!type.isNullOrEmpty()) " AND (${buildTypeQuery(type)})" else ""} ${if (!ids.isNullOrEmpty()) " AND ear.entity_id IN (:entities_ids)" else ""} + ${if (!includeDeleted) " AND deleted_at IS NULL" else ""} """.trimIndent() ) .let { @@ -443,6 +448,7 @@ class EntityAccessRightsService( return EntityAccessRights( id = toUri(row["entity_id"]), types = toList(row["types"]), + isDeleted = row["deleted_at"] != null, right = accessRight, specificAccessPolicy = toOptionalEnum(row["specific_access_policy"]) ) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/web/EntityAccessControlHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/web/EntityAccessControlHandler.kt index 996dd3985..cbbdf667c 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/authorization/web/EntityAccessControlHandler.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/authorization/web/EntityAccessControlHandler.kt @@ -4,10 +4,11 @@ import arrow.core.left import arrow.core.raise.either import com.egm.stellio.search.authorization.service.AuthorizationService import com.egm.stellio.search.authorization.service.EntityAccessRightsService +import com.egm.stellio.search.entity.model.FailedAttributeOperationResult import com.egm.stellio.search.entity.model.NotUpdatedDetails -import com.egm.stellio.search.entity.model.UpdateAttributeResult -import com.egm.stellio.search.entity.model.UpdateOperationResult -import com.egm.stellio.search.entity.model.updateResultFromDetailedResult +import com.egm.stellio.search.entity.model.OperationStatus +import com.egm.stellio.search.entity.model.SucceededAttributeOperationResult +import com.egm.stellio.search.entity.model.UpdateResult import com.egm.stellio.search.entity.util.composeEntitiesQueryFromGet import com.egm.stellio.shared.config.ApplicationProperties import com.egm.stellio.shared.model.AccessDeniedException @@ -19,6 +20,7 @@ import com.egm.stellio.shared.model.toNgsiLdAttribute import com.egm.stellio.shared.model.toNgsiLdAttributes import com.egm.stellio.shared.queryparameter.AllowedParameters import com.egm.stellio.shared.queryparameter.QP +import com.egm.stellio.shared.queryparameter.QueryParameter import com.egm.stellio.shared.util.AccessRight import com.egm.stellio.shared.util.AuthContextModel.ALL_ASSIGNABLE_IAM_RIGHTS import com.egm.stellio.shared.util.AuthContextModel.ALL_IAM_RIGHTS @@ -70,10 +72,11 @@ class EntityAccessControlHandler( @GetMapping("/entities", produces = [MediaType.APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE]) suspend fun getAuthorizedEntities( @RequestHeader httpHeaders: HttpHeaders, - @AllowedParameters(implemented = [QP.ID, QP.TYPE, QP.ATTRS, QP.COUNT, QP.OFFSET, QP.LIMIT]) + @AllowedParameters(implemented = [QP.ID, QP.TYPE, QP.ATTRS, QP.COUNT, QP.OFFSET, QP.LIMIT, QP.INCLUDE_DELETED]) @RequestParam queryParams: MultiValueMap ): ResponseEntity<*> = either { val sub = getSubFromSecurityContext() + val includeDeleted = queryParams.getFirst(QueryParameter.INCLUDE_DELETED.key)?.toBoolean() == true val contexts = getAuthzContextFromLinkHeaderOrDefault(httpHeaders, applicationProperties.contexts).bind() val mediaType = getApplicableMediaType(httpHeaders).bind() @@ -91,6 +94,7 @@ class EntityAccessControlHandler( val (count, entities) = authorizationService.getAuthorizedEntities( entitiesQuery, + includeDeleted, contexts, sub ).bind() @@ -254,24 +258,24 @@ class EntityAccessControlHandler( AccessRight.forAttributeName(ngsiLdRel.name).getOrNull()!! ).fold( ifLeft = { apiException -> - UpdateAttributeResult( + FailedAttributeOperationResult( ngsiLdRel.name, ngsiLdRelInstance.datasetId, - UpdateOperationResult.FAILED, + OperationStatus.FAILED, apiException.message ) }, ifRight = { - UpdateAttributeResult( + SucceededAttributeOperationResult( ngsiLdRel.name, ngsiLdRelInstance.datasetId, - UpdateOperationResult.APPENDED, - null + OperationStatus.APPENDED, + emptyMap() ) } ) } - val appendResult = updateResultFromDetailedResult(results) + val appendResult = UpdateResult(results) if (invalidAttributes.isEmpty() && unauthorizedInstances.isEmpty()) ResponseEntity.status(HttpStatus.NO_CONTENT).build() diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/discovery/service/AttributeService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/discovery/service/AttributeService.kt index eb9aecf56..c2d1720b6 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/discovery/service/AttributeService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/discovery/service/AttributeService.kt @@ -30,6 +30,7 @@ class AttributeService( """ SELECT DISTINCT(attribute_name) FROM temporal_entity_attribute + WHERE deleted_at IS NULL ORDER BY attribute_name """.trimIndent() ).allToMappedList { rowToAttributeNames(it) } @@ -42,7 +43,10 @@ class AttributeService( """ SELECT types, attribute_name FROM entity_payload - JOIN temporal_entity_attribute ON entity_payload.entity_id = temporal_entity_attribute.entity_id + JOIN temporal_entity_attribute + ON entity_payload.entity_id = temporal_entity_attribute.entity_id + AND temporal_entity_attribute.deleted_at IS NULL + WHERE entity_payload.deleted_at IS NULL ORDER BY attribute_name """.trimIndent() ).allToMappedList { rowToAttributeDetails(it) }.flatten().groupBy({ it.second }, { it.first }).toList() @@ -65,11 +69,14 @@ class AttributeService( WITH entities AS ( SELECT entity_id, attribute_name, attribute_type FROM temporal_entity_attribute - WHERE attribute_name = :attribute_name + WHERE attribute_name = :attribute_name + AND deleted_at IS NULL ) SELECT attribute_name, attribute_type, types, count(distinct(attribute_name)) as attribute_count FROM entity_payload - JOIN entities ON entity_payload.entity_id = entities.entity_id + JOIN entities + ON entity_payload.entity_id = entities.entity_id + AND entity_payload.deleted_at IS NULL GROUP BY types, attribute_name, attribute_type """.trimIndent() ) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/discovery/service/EntityTypeService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/discovery/service/EntityTypeService.kt index 07626ecb7..7a4d796c6 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/discovery/service/EntityTypeService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/discovery/service/EntityTypeService.kt @@ -29,6 +29,7 @@ class EntityTypeService( """ SELECT DISTINCT(unnest(types)) as type FROM entity_payload + WHERE deleted_at IS NULL ORDER BY type """.trimIndent() ).allToMappedList { rowToType(it) } @@ -41,7 +42,10 @@ class EntityTypeService( """ SELECT unnest(types) as type, attribute_name FROM entity_payload - JOIN temporal_entity_attribute ON entity_payload.entity_id = temporal_entity_attribute.entity_id + JOIN temporal_entity_attribute + ON entity_payload.entity_id = temporal_entity_attribute.entity_id + AND temporal_entity_attribute.deleted_at IS NULL + WHERE entity_payload.deleted_at IS NULL ORDER BY type """.trimIndent() ).allToMappedList { rowToEntityType(it) }.groupBy({ it.first }, { it.second }).toList() @@ -65,10 +69,12 @@ class EntityTypeService( SELECT entity_id FROM entity_payload WHERE :type_name = any (types) + AND deleted_at IS NULL ) SELECT attribute_name, attribute_type, (select count(entity_id) from entities) as entity_count FROM temporal_entity_attribute WHERE entity_id IN (SELECT entity_id FROM entities) + AND deleted_at IS NULL GROUP BY attribute_name, attribute_type """.trimIndent() ) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/listener/ObservationEventListener.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/listener/ObservationEventListener.kt index cb9a6f115..db1a4a7c9 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/listener/ObservationEventListener.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/listener/ObservationEventListener.kt @@ -3,6 +3,8 @@ package com.egm.stellio.search.entity.listener import arrow.core.Either import arrow.core.left import arrow.core.raise.either +import com.egm.stellio.search.entity.model.OperationStatus +import com.egm.stellio.search.entity.model.SucceededAttributeOperationResult import com.egm.stellio.search.entity.service.EntityEventService import com.egm.stellio.search.entity.service.EntityService import com.egm.stellio.shared.model.APIException @@ -120,9 +122,14 @@ class ObservationEventListener( entityEventService.publishAttributeChangeEvents( observationEvent.sub, observationEvent.entityId, - expandedAttribute.toExpandedAttributes(), - it, - false + listOf( + SucceededAttributeOperationResult( + observationEvent.attributeName, + observationEvent.datasetId, + OperationStatus.UPDATED, + expandedAttribute.toExpandedAttributes() + ) + ) ) } } @@ -143,7 +150,7 @@ class ObservationEventListener( entityService.appendAttributes( observationEvent.entityId, expandedAttribute.toExpandedAttributes(), - !observationEvent.overwrite, + false, observationEvent.sub ).map { if (it.notUpdated.isNotEmpty()) { @@ -157,9 +164,14 @@ class ObservationEventListener( entityEventService.publishAttributeChangeEvents( observationEvent.sub, observationEvent.entityId, - expandedAttribute.toExpandedAttributes(), - it, - observationEvent.overwrite + listOf( + SucceededAttributeOperationResult( + observationEvent.attributeName, + observationEvent.datasetId, + OperationStatus.APPENDED, + expandedAttribute.toExpandedAttributes() + ) + ) ) } } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/model/Attribute.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/model/Attribute.kt index bca913e99..d776b3572 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/model/Attribute.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/model/Attribute.kt @@ -1,18 +1,35 @@ package com.egm.stellio.search.entity.model import com.egm.stellio.shared.model.ExpandedTerm +import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_LANGUAGE +import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_LANGUAGEMAP_TERM +import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_OBJECT +import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE_TERM +import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_VALUE +import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_VALUE_TERM +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DATASET_ID_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_GEOPROPERTY_TYPE +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_GEOPROPERTY_VALUE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_GEOPROPERTY_VALUES import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_JSONPROPERTY_TYPE +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_JSONPROPERTY_VALUE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_JSONPROPERTY_VALUES import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_LANGUAGEPROPERTY_TYPE +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_LANGUAGEPROPERTY_VALUE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_LANGUAGEPROPERTY_VALUES +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_NONE_TERM +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_NULL import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_PROPERTY_TYPE +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_PROPERTY_VALUE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_PROPERTY_VALUES +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_RELATIONSHIP_OBJECT import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_RELATIONSHIP_OBJECTS import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_RELATIONSHIP_TYPE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_VOCABPROPERTY_TYPE +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_VOCABPROPERTY_VALUE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_VOCABPROPERTY_VALUES +import com.egm.stellio.shared.util.JsonLdUtils.buildNonReifiedPropertyValue +import com.egm.stellio.shared.util.JsonUtils.serializeObject import io.r2dbc.postgresql.codec.Json import org.springframework.data.annotation.Id import java.net.URI @@ -29,6 +46,7 @@ data class Attribute( val datasetId: URI? = null, val createdAt: ZonedDateTime, val modifiedAt: ZonedDateTime? = null, + val deletedAt: ZonedDateTime? = null, val payload: Json ) { enum class AttributeValueType { @@ -63,6 +81,21 @@ data class Attribute( VocabProperty -> NGSILD_VOCABPROPERTY_TYPE.uri } + /** + * Returns the expanded name of the member who holds the value of the attribute. + * + * For instance, https://uri.etsi.org/ngsi-ld/hasJSON if it is a JsonProperty + */ + fun toExpandedValueMember(): String = + when (this) { + Property -> NGSILD_PROPERTY_VALUE + Relationship -> NGSILD_RELATIONSHIP_OBJECT + GeoProperty -> NGSILD_GEOPROPERTY_VALUE + JsonProperty -> NGSILD_JSONPROPERTY_VALUE + LanguageProperty -> NGSILD_LANGUAGEPROPERTY_VALUE + VocabProperty -> NGSILD_VOCABPROPERTY_VALUE + } + /** * Returns the key of the member for the simplified representation of the attribute, as defined in 4.5.9 */ @@ -75,5 +108,37 @@ data class Attribute( LanguageProperty -> NGSILD_LANGUAGEPROPERTY_VALUES VocabProperty -> NGSILD_VOCABPROPERTY_VALUES } + + fun toNullCompactedRepresentation(datasetId: URI? = null): Map = + when (this) { + Property, GeoProperty, JsonProperty, VocabProperty -> + mapOf( + JSONLD_TYPE_TERM to this.name, + JSONLD_VALUE_TERM to NGSILD_NULL + ) + Relationship -> + mapOf( + JSONLD_TYPE_TERM to this.name, + JSONLD_OBJECT to NGSILD_NULL + ) + LanguageProperty -> + mapOf( + JSONLD_TYPE_TERM to this.name, + JSONLD_LANGUAGEMAP_TERM to mapOf(NGSILD_NONE_TERM to NGSILD_NULL) + ) + }.let { nullAttrRepresentation -> + if (datasetId != null) + nullAttrRepresentation.plus( + NGSILD_DATASET_ID_PROPERTY to buildNonReifiedPropertyValue(datasetId.toString()) + ) + else nullAttrRepresentation + } + + fun toNullValue(): String = + when (this) { + Property, GeoProperty, JsonProperty, VocabProperty, Relationship -> NGSILD_NULL + LanguageProperty -> + serializeObject(listOf(mapOf(JSONLD_VALUE to NGSILD_NULL, JSONLD_LANGUAGE to NGSILD_NONE_TERM))) + } } } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/model/Entity.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/model/Entity.kt index e81816479..9461f2860 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/model/Entity.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/model/Entity.kt @@ -1,5 +1,6 @@ package com.egm.stellio.search.entity.model +import com.egm.stellio.shared.model.ExpandedEntity import com.egm.stellio.shared.model.ExpandedTerm import com.egm.stellio.shared.util.AuthContextModel import com.egm.stellio.shared.util.AuthContextModel.SpecificAccessPolicy @@ -7,6 +8,7 @@ import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_ID import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_VALUE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_CREATED_AT_PROPERTY +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DELETED_AT_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_MODIFIED_AT_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_SCOPE_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.buildExpandedPropertyValue @@ -21,6 +23,7 @@ data class Entity( val scopes: List? = null, val createdAt: ZonedDateTime, val modifiedAt: ZonedDateTime? = null, + val deletedAt: ZonedDateTime? = null, val payload: Json, val specificAccessPolicy: SpecificAccessPolicy? = null ) { @@ -44,4 +47,20 @@ data class Entity( return resultEntity } + + fun toExpandedDeletedEntity( + deletedAt: ZonedDateTime + ): ExpandedEntity = + ExpandedEntity( + members = mapOf( + JSONLD_ID to entityId, + JSONLD_TYPE to types, + NGSILD_CREATED_AT_PROPERTY to buildNonReifiedTemporalValue(createdAt), + NGSILD_DELETED_AT_PROPERTY to buildNonReifiedTemporalValue(deletedAt), + ).run { + if (modifiedAt != null) + this.plus(NGSILD_MODIFIED_AT_PROPERTY to buildNonReifiedTemporalValue(modifiedAt)) + else this + } + ) } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/model/UpdateResult.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/model/UpdateResult.kt index 9ccf6a575..a34eef570 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/model/UpdateResult.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/model/UpdateResult.kt @@ -1,78 +1,88 @@ package com.egm.stellio.search.entity.model +import com.egm.stellio.shared.model.ExpandedAttributeInstance import com.fasterxml.jackson.annotation.JsonIgnore -import com.fasterxml.jackson.annotation.JsonValue import java.net.URI +/** + * UpdateResult datatype as defined in 5.2.18 + */ data class UpdateResult( - val updated: List, + val updated: List, val notUpdated: List ) { @JsonIgnore fun isSuccessful(): Boolean = - notUpdated.isEmpty() && - updated.all { it.updateOperationResult.isSuccessResult() } + notUpdated.isEmpty() - @JsonIgnore - fun mergeWith(other: UpdateResult): UpdateResult = - UpdateResult( - updated = this.updated.plus(other.updated), - notUpdated = this.notUpdated.plus(other.notUpdated) - ) + companion object { - @JsonIgnore - fun hasSuccessfulUpdate(): Boolean = - this.updated.isNotEmpty() + operator fun invoke(operationsResults: List): UpdateResult = + operationsResults.map { + when (it) { + is SucceededAttributeOperationResult -> it.attributeName + is FailedAttributeOperationResult -> NotUpdatedDetails(it.attributeName, it.errorMessage) + } + }.let { + UpdateResult( + it.filterIsInstance(), + it.filterIsInstance() + ) + } + } } val EMPTY_UPDATE_RESULT: UpdateResult = UpdateResult(emptyList(), emptyList()) +/** + * NotUpdatedDetails as defined in 5.2.19 + */ data class NotUpdatedDetails( val attributeName: String, val reason: String ) -data class UpdatedDetails( - @JsonValue - val attributeName: String, - @JsonIgnore - val datasetId: URI?, - @JsonIgnore - val updateOperationResult: UpdateOperationResult +/** + * Internal structure used to convey the result of an operation (update, delete...) + */ +sealed class AttributeOperationResult( + open val attributeName: String, + open val datasetId: URI? = null, + open val operationStatus: OperationStatus ) -data class UpdateAttributeResult( - val attributeName: String, - val datasetId: URI? = null, - val updateOperationResult: UpdateOperationResult, - val errorMessage: String? = null -) { - fun isSuccessfullyUpdated() = - this.updateOperationResult in listOf( - UpdateOperationResult.APPENDED, - UpdateOperationResult.REPLACED, - UpdateOperationResult.UPDATED, - UpdateOperationResult.IGNORED - ) -} +data class SucceededAttributeOperationResult( + override val attributeName: String, + override val datasetId: URI? = null, + override val operationStatus: OperationStatus, + val newExpandedValue: ExpandedAttributeInstance, +) : AttributeOperationResult(attributeName, datasetId, operationStatus) + +data class FailedAttributeOperationResult( + override val attributeName: String, + override val datasetId: URI? = null, + override val operationStatus: OperationStatus, + val errorMessage: String +) : AttributeOperationResult(attributeName, datasetId, operationStatus) -enum class UpdateOperationResult { +enum class OperationStatus { APPENDED, REPLACED, UPDATED, + DELETED, IGNORED, FAILED; - fun isSuccessResult(): Boolean = listOf(APPENDED, REPLACED, UPDATED).contains(this) -} + fun isSuccessResult(): Boolean = getSuccessStatuses().contains(this) -fun updateResultFromDetailedResult(updateStatuses: List): UpdateResult { - val updated = updateStatuses.filter { it.isSuccessfullyUpdated() } - .map { UpdatedDetails(it.attributeName, it.datasetId, it.updateOperationResult) } + companion object { + fun getSuccessStatuses(): List = listOf(APPENDED, REPLACED, UPDATED, DELETED, IGNORED) + } +} - val notUpdated = updateStatuses.filter { !it.isSuccessfullyUpdated() } - .map { NotUpdatedDetails(it.attributeName, it.errorMessage!!) } +fun List.hasSuccessfulResult(): Boolean = + this.any { it is SucceededAttributeOperationResult } - return UpdateResult(updated, notUpdated) -} +fun List.getSucceededOperations(): List = + this.filterIsInstance() diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityAttributeService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityAttributeService.kt index adf1ff565..223cec0e4 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityAttributeService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityAttributeService.kt @@ -22,12 +22,13 @@ import com.egm.stellio.search.common.util.valueToDoubleOrNull import com.egm.stellio.search.common.util.valueToStringOrNull import com.egm.stellio.search.entity.model.Attribute import com.egm.stellio.search.entity.model.AttributeMetadata +import com.egm.stellio.search.entity.model.AttributeOperationResult import com.egm.stellio.search.entity.model.EntitiesQuery -import com.egm.stellio.search.entity.model.UpdateAttributeResult -import com.egm.stellio.search.entity.model.UpdateOperationResult -import com.egm.stellio.search.entity.model.UpdateResult -import com.egm.stellio.search.entity.model.updateResultFromDetailedResult +import com.egm.stellio.search.entity.model.FailedAttributeOperationResult +import com.egm.stellio.search.entity.model.OperationStatus +import com.egm.stellio.search.entity.model.SucceededAttributeOperationResult import com.egm.stellio.search.entity.util.guessAttributeValueType +import com.egm.stellio.search.entity.util.hasNgsiLdNullValue import com.egm.stellio.search.entity.util.mergePatch import com.egm.stellio.search.entity.util.partialUpdatePatch import com.egm.stellio.search.entity.util.prepareAttributes @@ -35,6 +36,7 @@ import com.egm.stellio.search.entity.util.toAttributeMetadata import com.egm.stellio.search.entity.util.toExpandedAttributeInstance import com.egm.stellio.search.temporal.model.AttributeInstance import com.egm.stellio.search.temporal.service.AttributeInstanceService +import com.egm.stellio.shared.config.ApplicationProperties import com.egm.stellio.shared.model.APIException import com.egm.stellio.shared.model.BadRequestDataException import com.egm.stellio.shared.model.ExpandedAttribute @@ -46,6 +48,7 @@ import com.egm.stellio.shared.model.NgsiLdAttribute import com.egm.stellio.shared.model.NgsiLdEntity import com.egm.stellio.shared.model.ResourceNotFoundException import com.egm.stellio.shared.model.WKTCoordinates +import com.egm.stellio.shared.model.addSysAttrs import com.egm.stellio.shared.model.flatOnInstances import com.egm.stellio.shared.model.getAttributeFromExpandedAttributes import com.egm.stellio.shared.model.getDatasetId @@ -56,6 +59,7 @@ import com.egm.stellio.shared.model.isAttributeOfType import com.egm.stellio.shared.model.toNgsiLdEntity import com.egm.stellio.shared.util.AttributeType import com.egm.stellio.shared.util.AuthContextModel +import com.egm.stellio.shared.util.JsonLdUtils import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_JSONPROPERTY_VALUE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_LANGUAGEPROPERTY_VALUE @@ -70,6 +74,7 @@ import com.egm.stellio.shared.util.JsonUtils.serializeObject import com.egm.stellio.shared.util.Sub import com.egm.stellio.shared.util.attributeNotFoundMessage import com.egm.stellio.shared.util.entityNotFoundMessage +import com.egm.stellio.shared.util.ngsiLdDateTime import io.r2dbc.postgresql.codec.Json import org.slf4j.LoggerFactory import org.springframework.r2dbc.core.DatabaseClient @@ -77,14 +82,14 @@ import org.springframework.r2dbc.core.bind import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional import java.net.URI -import java.time.ZoneOffset import java.time.ZonedDateTime -import java.util.UUID +import java.util.* @Service class EntityAttributeService( private val databaseClient: DatabaseClient, - private val attributeInstanceService: AttributeInstanceService + private val attributeInstanceService: AttributeInstanceService, + private val applicationProperties: ApplicationProperties ) { private val logger = LoggerFactory.getLogger(javaClass) @@ -99,6 +104,12 @@ class EntityAttributeService( VALUES (:id, :entity_id, :attribute_name, :attribute_type, :attribute_value_type, :created_at, :dataset_id, :payload) + ON CONFLICT (entity_id, attribute_name, dataset_id) + DO UPDATE SET deleted_at = null, + attribute_type = :attribute_type, + attribute_value_type = :attribute_value_type, + modified_at = :created_at, + payload = :payload """.trimIndent() ) .bind("id", attribute.id) @@ -111,31 +122,6 @@ class EntityAttributeService( .bind("payload", attribute.payload) .execute() - @Transactional - suspend fun updateOnReplace( - attributeUUID: UUID, - attributeMetadata: AttributeMetadata, - modifiedAt: ZonedDateTime, - payload: String - ): Either = - databaseClient.sql( - """ - UPDATE temporal_entity_attribute - SET - attribute_type = :attribute_type, - attribute_value_type = :attribute_value_type, - modified_at = :modified_at, - payload = :payload - WHERE id = :id - """.trimIndent() - ) - .bind("id", attributeUUID) - .bind("attribute_type", attributeMetadata.type.toString()) - .bind("attribute_value_type", attributeMetadata.valueType.toString()) - .bind("modified_at", modifiedAt) - .bind("payload", Json.of(payload)) - .execute() - @Transactional suspend fun updateOnUpdate( attributeUUID: UUID, @@ -169,7 +155,7 @@ class EntityAttributeService( contexts: List, sub: String? = null ): Either = either { - val createdAt = ZonedDateTime.now(ZoneOffset.UTC) + val createdAt = ngsiLdDateTime() val expandedEntity = expandJsonLdEntity(payload, contexts) val ngsiLdEntity = expandedEntity.toNgsiLdEntity().bind() ngsiLdEntity.prepareAttributes() @@ -218,40 +204,39 @@ class EntityAttributeService( createdAt: ZonedDateTime, attributePayload: ExpandedAttributeInstance, sub: Sub? - ): Either = - either { - logger.debug("Adding attribute {} to entity {}", attributeName, entityId) - val attribute = Attribute( - entityId = entityId, - attributeName = attributeName, - attributeType = attributeMetadata.type, - attributeValueType = attributeMetadata.valueType, - datasetId = attributeMetadata.datasetId, - createdAt = createdAt, - payload = Json.of(serializeObject(attributePayload)) - ) - create(attribute).bind() + ): Either = either { + logger.debug("Adding attribute {} to entity {}", attributeName, entityId) + val attribute = Attribute( + entityId = entityId, + attributeName = attributeName, + attributeType = attributeMetadata.type, + attributeValueType = attributeMetadata.valueType, + datasetId = attributeMetadata.datasetId, + createdAt = createdAt, + payload = Json.of(serializeObject(attributePayload)) + ) + create(attribute).bind() + + val attributeInstance = AttributeInstance( + attributeUuid = attribute.id, + timeProperty = AttributeInstance.TemporalProperty.CREATED_AT, + time = createdAt, + attributeMetadata = attributeMetadata, + payload = attributePayload, + sub = sub + ) + attributeInstanceService.create(attributeInstance).bind() - val attributeInstance = AttributeInstance( + if (attributeMetadata.observedAt != null) { + val attributeObservedAtInstance = AttributeInstance( attributeUuid = attribute.id, - timeProperty = AttributeInstance.TemporalProperty.CREATED_AT, - time = createdAt, + time = attributeMetadata.observedAt, attributeMetadata = attributeMetadata, - payload = attributePayload, - sub = sub + payload = attributePayload ) - attributeInstanceService.create(attributeInstance).bind() - - if (attributeMetadata.observedAt != null) { - val attributeObservedAtInstance = AttributeInstance( - attributeUuid = attribute.id, - time = attributeMetadata.observedAt, - attributeMetadata = attributeMetadata, - payload = attributePayload - ) - attributeInstanceService.create(attributeObservedAtInstance).bind() - } + attributeInstanceService.create(attributeObservedAtInstance).bind() } + } @Transactional suspend fun replaceAttribute( @@ -268,32 +253,15 @@ class EntityAttributeService( attributeMetadata.datasetId, attribute.entityId ) - updateOnReplace( - attribute.id, + deleteAttribute(attribute.entityId, attribute.attributeName, attribute.datasetId, false, createdAt).bind() + addAttribute( + attribute.entityId, + attribute.attributeName, attributeMetadata, createdAt, - serializeObject(attributePayload) + attributePayload, + sub ).bind() - - val attributeInstance = AttributeInstance( - attributeUuid = attribute.id, - timeProperty = AttributeInstance.TemporalProperty.MODIFIED_AT, - time = createdAt, - attributeMetadata = attributeMetadata, - payload = attributePayload, - sub = sub - ) - attributeInstanceService.create(attributeInstance).bind() - - if (attributeMetadata.observedAt != null) { - val attributeObservedAtInstance = AttributeInstance( - attributeUuid = attribute.id, - time = attributeMetadata.observedAt, - attributeMetadata = attributeMetadata, - payload = attributePayload - ) - attributeInstanceService.create(attributeObservedAtInstance).bind() - } } @Transactional @@ -329,22 +297,9 @@ class EntityAttributeService( } @Transactional - suspend fun deleteAttributes(entityId: URI): Either { - val uuids = databaseClient.sql( - """ - DELETE FROM temporal_entity_attribute - WHERE entity_id = :entity_id - RETURNING id - """.trimIndent() - ) - .bind("entity_id", entityId) - .allToMappedList { - toUuid(it["id"]) - } - - return if (uuids.isNotEmpty()) - attributeInstanceService.deleteInstancesOfEntity(uuids) - else Unit.right() + suspend fun deleteAttributes(entityId: URI, deletedAt: ZonedDateTime): Either = either { + val attributesToDelete = getForEntity(entityId, emptySet(), emptySet()) + deleteSelectedAttributes(attributesToDelete, deletedAt).bind() } @Transactional @@ -352,56 +307,129 @@ class EntityAttributeService( entityId: URI, attributeName: String, datasetId: URI?, - deleteAll: Boolean = false - ): Either = - either { - logger.debug("Deleting attribute {} from entity {} (all: {})", attributeName, entityId, deleteAll) - if (deleteAll) { - attributeInstanceService.deleteAllInstancesOfAttribute(entityId, attributeName).bind() - deleteAllInstances(entityId, attributeName).bind() - } else { - attributeInstanceService.deleteInstancesOfAttribute(entityId, attributeName, datasetId).bind() - deleteSpecificInstance(entityId, attributeName, datasetId).bind() + deleteAll: Boolean = false, + deletedAt: ZonedDateTime + ): Either> = either { + logger.debug("Deleting attribute {} from entity {} (all: {})", attributeName, entityId, deleteAll) + val attributesToDelete = + if (deleteAll) + getForEntity(entityId, setOf(attributeName), emptySet()) + else + listOf(getForEntityAndAttribute(entityId, attributeName, datasetId).bind()) + + deleteSelectedAttributes(attributesToDelete, deletedAt).bind() + .map { expandedAttributeInstance -> + SucceededAttributeOperationResult( + attributeName, + datasetId, + OperationStatus.DELETED, + expandedAttributeInstance + ) } + } + + @Transactional + internal suspend fun deleteSelectedAttributes( + attributesToDelete: List, + deletedAt: ZonedDateTime + ): Either> = either { + if (attributesToDelete.isEmpty()) return emptyList().right() + val attributesToDeleteWithPayload = attributesToDelete.map { + Pair( + it, + JsonLdUtils.expandAttribute( + it.attributeName, + it.attributeType.toNullCompactedRepresentation(it.datasetId), + listOf(applicationProperties.contexts.core) + ).second[0] + ) } + val teasTimestamps = databaseClient.sql( + """ + UPDATE temporal_entity_attribute + SET deleted_at = new.deleted_at, + payload = new.payload + FROM (VALUES :values) AS new(uuid, deleted_at, payload) + WHERE temporal_entity_attribute.id = new.uuid + RETURNING id, created_at, modified_at, new.deleted_at + """.trimIndent() + ) + .bind("values", attributesToDeleteWithPayload.map { arrayOf(it.first.id, deletedAt, it.second.toJson()) }) + .allToMappedList { row -> + mapOf( + toUuid(row["id"]) to Triple( + toZonedDateTime(row["created_at"]), + toOptionalZonedDateTime(row["modified_at"]), + toZonedDateTime(row["deleted_at"]) + ) + ) + } + + attributesToDeleteWithPayload.forEach { (attribute, expandedAttributePayload) -> + attributeInstanceService.addDeletedAttributeInstance( + attributeUuid = attribute.id, + value = attribute.attributeType.toNullValue(), + deletedAt = deletedAt, + attributeValues = expandedAttributePayload + ).bind() + } + + attributesToDeleteWithPayload.map { (attribute, expandedAttributeInstance) -> + val teaTimestamps = teasTimestamps.find { it.containsKey(attribute.id) }!!.values.first() + expandedAttributeInstance.addSysAttrs(true, teaTimestamps.first, teaTimestamps.second, teaTimestamps.third) + } + } + @Transactional - suspend fun deleteSpecificInstance( + suspend fun permanentlyDeleteAttribute( entityId: URI, attributeName: String, - datasetId: URI? - ): Either = + datasetId: URI?, + deleteAll: Boolean = false + ): Either = either { + logger.debug("Permanently deleting attribute {} from entity {} (all: {})", attributeName, entityId, deleteAll) + val attributesToDelete = + if (deleteAll) + getForEntity(entityId, setOf(attributeName), emptySet(), false) + else + listOf(getForEntityAndAttribute(entityId, attributeName, datasetId).bind()) + databaseClient.sql( """ DELETE FROM temporal_entity_attribute - WHERE entity_id = :entity_id - ${datasetId.toDatasetIdFilter()} - AND attribute_name = :attribute_name + WHERE id IN(:uuids) """.trimIndent() ) - .bind("entity_id", entityId) - .bind("attribute_name", attributeName) - .let { - if (datasetId != null) it.bind("dataset_id", datasetId) - else it - } + .bind("uuids", attributesToDelete.map { it.id }) .execute() + if (deleteAll) + attributeInstanceService.deleteAllInstancesOfAttribute(entityId, attributeName).bind() + else + attributeInstanceService.deleteInstancesOfAttribute(entityId, attributeName, datasetId).bind() + } + @Transactional - suspend fun deleteAllInstances( + suspend fun permanentlyDeleteAttributes( entityId: URI, - attributeName: String - ): Either = - databaseClient.sql( + ): Either = either { + logger.debug("Permanently deleting all attributes from entity {}", entityId) + + val deletedTeas = databaseClient.sql( """ DELETE FROM temporal_entity_attribute WHERE entity_id = :entity_id - AND attribute_name = :attribute_name + RETURNING id """.trimIndent() ) .bind("entity_id", entityId) - .bind("attribute_name", attributeName) - .execute() + .allToMappedList { toUuid(it["id"]) } + + if (deletedTeas.isNotEmpty()) + attributeInstanceService.deleteInstancesOfEntity(deletedTeas).bind() + else Unit.right() + } suspend fun getForEntities( entitiesIds: List, @@ -426,7 +454,7 @@ class EntityAttributeService( val selectQuery = """ SELECT id, entity_id, attribute_name, attribute_type, attribute_value_type, created_at, modified_at, - dataset_id, payload + deleted_at, dataset_id, payload FROM temporal_entity_attribute WHERE entity_id IN (:entities_ids) $filterOnAttributes @@ -440,7 +468,12 @@ class EntityAttributeService( .allToMappedList { rowToAttribute(it) } } - suspend fun getForEntity(id: URI, attrs: Set, datasetIds: Set): List { + suspend fun getForEntity( + id: URI, + attrs: Set, + datasetIds: Set, + excludedDeleted: Boolean = true + ): List { val filterOnAttributes = if (attrs.isNotEmpty()) " AND " + attrs.joinToString( @@ -463,6 +496,7 @@ class EntityAttributeService( dataset_id, payload FROM temporal_entity_attribute WHERE entity_id = :entity_id + ${if (excludedDeleted) " and deleted_at is null " else ""} $filterOnAttributes $filterOnDatasetId """.trimIndent() @@ -500,33 +534,6 @@ class EntityAttributeService( } } - suspend fun hasAttribute( - id: URI, - attributeName: String, - datasetId: URI? = null - ): Either { - val selectQuery = - """ - SELECT count(entity_id) as count - FROM temporal_entity_attribute - WHERE entity_id = :entity_id - ${datasetId.toDatasetIdFilter()} - AND attribute_name = :attribute_name - """.trimIndent() - - return databaseClient - .sql(selectQuery) - .bind("entity_id", id) - .bind("attribute_name", attributeName) - .let { - if (datasetId != null) it.bind("dataset_id", datasetId) - else it - } - .oneToResult { - it["count"] as Long == 1L - } - } - private fun rowToAttribute(row: Map) = Attribute( id = toUuid(row["id"]), @@ -539,6 +546,7 @@ class EntityAttributeService( datasetId = toOptionalUri(row["dataset_id"]), createdAt = toZonedDateTime(row["created_at"]), modifiedAt = toOptionalZonedDateTime(row["modified_at"]), + deletedAt = toOptionalZonedDateTime(row["deleted_at"]), payload = toJson(row["payload"]) ) @@ -564,6 +572,7 @@ class EntityAttributeService( from temporal_entity_attribute where entity_id = :entity_id and attribute_name = :attribute_name + and deleted_at is null $datasetIdFilter ) as attributeNameExists; """.trimIndent() @@ -594,7 +603,7 @@ class EntityAttributeService( disallowOverwrite: Boolean, createdAt: ZonedDateTime, sub: Sub? - ): Either = either { + ): Either> = either { val attributeInstances = ngsiLdAttributes.flatOnInstances() attributeInstances.parMap { (ngsiLdAttribute, ngsiLdAttributeInstance) -> logger.debug("Appending attribute {} in entity {}", ngsiLdAttribute.name, entityUri) @@ -615,21 +624,20 @@ class EntityAttributeService( attributePayload, sub ).map { - UpdateAttributeResult( + SucceededAttributeOperationResult( ngsiLdAttribute.name, ngsiLdAttributeInstance.datasetId, - UpdateOperationResult.APPENDED, - null + OperationStatus.APPENDED, + attributePayload ) }.bind() } else if (disallowOverwrite) { - val message = "Attribute already exists on $entityUri and overwrite is not allowed, ignoring" - logger.info(message) - UpdateAttributeResult( + logger.info("Attribute already exists on $entityUri and overwrite is not allowed, ignoring") + SucceededAttributeOperationResult( ngsiLdAttribute.name, ngsiLdAttributeInstance.datasetId, - UpdateOperationResult.IGNORED, - message + OperationStatus.IGNORED, + attributePayload ).right().bind() } else { replaceAttribute( @@ -640,16 +648,16 @@ class EntityAttributeService( attributePayload, sub ).map { - UpdateAttributeResult( + SucceededAttributeOperationResult( ngsiLdAttribute.name, ngsiLdAttributeInstance.datasetId, - UpdateOperationResult.REPLACED, - null + OperationStatus.REPLACED, + attributePayload ) }.bind() } } - }.fold({ it.left() }, { updateResultFromDetailedResult(it).right() }) + }.fold({ it.left() }, { it.right() }) @Transactional suspend fun updateAttributes( @@ -658,7 +666,7 @@ class EntityAttributeService( expandedAttributes: ExpandedAttributes, createdAt: ZonedDateTime, sub: Sub? - ): Either = either { + ): Either> = either { val attributeInstances = ngsiLdAttributes.flatOnInstances() attributeInstances.parMap { (ngsiLdAttribute, ngsiLdAttributeInstance) -> logger.debug("Updating attribute {} in entity {}", ngsiLdAttribute.name, entityUri) @@ -670,41 +678,50 @@ class EntityAttributeService( ngsiLdAttribute.name, ngsiLdAttributeInstance.datasetId )!! - if (currentAttribute != null) { - replaceAttribute( - currentAttribute, - ngsiLdAttribute, + + if (currentAttribute == null) { + addAttribute( + entityUri, + ngsiLdAttribute.name, attributeMetadata, createdAt, attributePayload, sub ).map { - UpdateAttributeResult( + SucceededAttributeOperationResult( ngsiLdAttribute.name, ngsiLdAttributeInstance.datasetId, - UpdateOperationResult.REPLACED, - null + OperationStatus.APPENDED, + attributePayload ) }.bind() - } else { - addAttribute( + } else if (hasNgsiLdNullValue(attributePayload, currentAttribute.attributeType)) { + deleteAttribute( entityUri, ngsiLdAttribute.name, + ngsiLdAttributeInstance.datasetId, + false, + createdAt + ).bind().first() + } else { + replaceAttribute( + currentAttribute, + ngsiLdAttribute, attributeMetadata, createdAt, attributePayload, sub ).map { - UpdateAttributeResult( + SucceededAttributeOperationResult( ngsiLdAttribute.name, ngsiLdAttributeInstance.datasetId, - UpdateOperationResult.APPENDED, - null + OperationStatus.REPLACED, + attributePayload ) }.bind() } } - }.fold({ it.left() }, { updateResultFromDetailedResult(it).right() }) + }.fold({ it.left() }, { it.right() }) @Transactional suspend fun partialUpdateAttribute( @@ -712,20 +729,30 @@ class EntityAttributeService( expandedAttribute: ExpandedAttribute, modifiedAt: ZonedDateTime, sub: Sub? - ): Either = either { + ): Either = either { val attributeName = expandedAttribute.first val attributeValues = expandedAttribute.second[0] - logger.debug( - "Updating attribute {} of entity {} with values: {}", - attributeName, - entityId, - attributeValues - ) + logger.debug("Partial updating attribute {} in entity {}", attributeName, entityId) val datasetId = attributeValues.getDatasetId() - val exists = hasAttribute(entityId, attributeName, datasetId).bind() - val updateAttributeResult = - if (exists) { + val currentAttribute = getForEntityAndAttribute(entityId, attributeName, datasetId).fold({ null }, { it }) + val attributeOperationResult = + if (currentAttribute == null) { + FailedAttributeOperationResult( + attributeName, + datasetId, + OperationStatus.FAILED, + "Unknown attribute $attributeName with datasetId $datasetId in entity $entityId" + ) + } else if (hasNgsiLdNullValue(attributeValues, currentAttribute.attributeType)) { + deleteAttribute( + entityId, + attributeName, + datasetId, + false, + modifiedAt + ).bind().first() + } else { // first update payload in temporal entity attribute val attribute = getForEntityAndAttribute(entityId, attributeName, datasetId).bind() attributeValues[JSONLD_TYPE]?.let { @@ -749,22 +776,15 @@ class EntityAttributeService( ) attributeInstanceService.create(attributeInstance).bind() - UpdateAttributeResult( + SucceededAttributeOperationResult( attributeName, datasetId, - UpdateOperationResult.UPDATED, - null - ) - } else { - UpdateAttributeResult( - attributeName, - datasetId, - UpdateOperationResult.FAILED, - "Unknown attribute $attributeName with datasetId $datasetId in entity $entityId" + OperationStatus.UPDATED, + updatedAttributeInstance ) } - updateResultFromDetailedResult(listOf(updateAttributeResult)) + attributeOperationResult } @Transactional @@ -802,7 +822,7 @@ class EntityAttributeService( ).bind() } else { logger.debug("Adding instance to attribute {} to entity {}", currentAttribute.attributeName, entityUri) - attributeInstanceService.addAttributeInstance( + attributeInstanceService.addObservedAttributeInstance( currentAttribute.id, attributeMetadata, expandedAttributes[currentAttribute.attributeName]!!.first() @@ -818,7 +838,7 @@ class EntityAttributeService( createdAt: ZonedDateTime, observedAt: ZonedDateTime?, sub: Sub? - ): Either = either { + ): Either> = either { val attributeInstances = ngsiLdAttributes.flatOnInstances() attributeInstances.parMap { (ngsiLdAttribute, ngsiLdAttributeInstance) -> logger.debug("Merging attribute {} in entity {}", ngsiLdAttribute.name, entityUri) @@ -831,7 +851,7 @@ class EntityAttributeService( ngsiLdAttributeInstance.datasetId )!! - if (currentAttribute == null) { + if (currentAttribute == null) addAttribute( entityUri, ngsiLdAttribute.name, @@ -840,14 +860,22 @@ class EntityAttributeService( attributePayload, sub ).map { - UpdateAttributeResult( + SucceededAttributeOperationResult( ngsiLdAttribute.name, ngsiLdAttributeInstance.datasetId, - UpdateOperationResult.APPENDED, - null + OperationStatus.APPENDED, + attributePayload ) }.bind() - } else { + else if (hasNgsiLdNullValue(attributePayload, currentAttribute.attributeType)) + deleteAttribute( + entityUri, + ngsiLdAttribute.name, + ngsiLdAttributeInstance.datasetId, + false, + createdAt + ).bind().first() + else mergeAttribute( currentAttribute, ngsiLdAttribute.name, @@ -857,16 +885,15 @@ class EntityAttributeService( attributePayload, sub ).map { - UpdateAttributeResult( + SucceededAttributeOperationResult( ngsiLdAttribute.name, ngsiLdAttributeInstance.datasetId, - UpdateOperationResult.UPDATED, - null + OperationStatus.UPDATED, + attributePayload ) }.bind() - } } - }.fold({ it.left() }, { updateResultFromDetailedResult(it).right() }) + }.fold({ it.left() }, { it.right() }) @Transactional suspend fun replaceAttribute( @@ -875,19 +902,19 @@ class EntityAttributeService( expandedAttribute: ExpandedAttribute, replacedAt: ZonedDateTime, sub: Sub? - ): Either = either { + ): Either = either { val ngsiLdAttributeInstance = ngsiLdAttribute.getAttributeInstances()[0] val attributeName = ngsiLdAttribute.name val datasetId = ngsiLdAttributeInstance.datasetId val currentTea = getForEntityAndAttribute(entityId, attributeName, datasetId).fold({ null }, { it }) val attributeMetadata = ngsiLdAttributeInstance.toAttributeMetadata().bind() - val updateAttributeResult = + val attributeOperationResult = if (currentTea == null) { - UpdateAttributeResult( + FailedAttributeOperationResult( attributeName, datasetId, - UpdateOperationResult.FAILED, + OperationStatus.FAILED, "Unknown attribute $attributeName with datasetId $datasetId in entity $entityId" ) } else { @@ -900,15 +927,15 @@ class EntityAttributeService( sub ).bind() - UpdateAttributeResult( + SucceededAttributeOperationResult( ngsiLdAttribute.name, ngsiLdAttributeInstance.datasetId, - UpdateOperationResult.REPLACED, - null + OperationStatus.REPLACED, + expandedAttribute.second.first() ) } - updateResultFromDetailedResult(listOf(updateAttributeResult)) + attributeOperationResult } suspend fun getValueFromPartialAttributePayload( diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityEventService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityEventService.kt index 8586746ef..830a91a7b 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityEventService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityEventService.kt @@ -2,12 +2,10 @@ package com.egm.stellio.search.entity.service import arrow.core.Either import com.egm.stellio.search.entity.model.Entity -import com.egm.stellio.search.entity.model.UpdateOperationResult -import com.egm.stellio.search.entity.model.UpdateResult -import com.egm.stellio.search.entity.model.UpdatedDetails +import com.egm.stellio.search.entity.model.OperationStatus +import com.egm.stellio.search.entity.model.SucceededAttributeOperationResult import com.egm.stellio.shared.model.APIException import com.egm.stellio.shared.model.AttributeAppendEvent -import com.egm.stellio.shared.model.AttributeDeleteAllInstancesEvent import com.egm.stellio.shared.model.AttributeDeleteEvent import com.egm.stellio.shared.model.AttributeReplaceEvent import com.egm.stellio.shared.model.AttributeUpdateEvent @@ -16,10 +14,10 @@ import com.egm.stellio.shared.model.EntityDeleteEvent import com.egm.stellio.shared.model.EntityEvent import com.egm.stellio.shared.model.EntityReplaceEvent import com.egm.stellio.shared.model.EventsType -import com.egm.stellio.shared.model.ExpandedAttributes +import com.egm.stellio.shared.model.ExpandedAttributeInstance +import com.egm.stellio.shared.model.ExpandedEntity import com.egm.stellio.shared.model.ExpandedTerm -import com.egm.stellio.shared.model.getAttributeFromExpandedAttributes -import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE +import com.egm.stellio.shared.util.JsonUtils.deserializeAsMap import com.egm.stellio.shared.util.JsonUtils.serializeObject import com.egm.stellio.shared.util.getTenantFromContext import kotlinx.coroutines.CoroutineScope @@ -84,18 +82,20 @@ class EntityEventService( suspend fun publishEntityDeleteEvent( sub: String?, - entity: Entity + previousEntity: Entity, + deletedEntityPayload: ExpandedEntity ): Job { val tenantName = getTenantFromContext() return coroutineScope.launch { - logger.debug("Sending delete event for entity {} in tenant {}", entity.entityId, tenantName) + logger.debug("Sending delete event for entity {} in tenant {}", previousEntity.entityId, tenantName) publishEntityEvent( EntityDeleteEvent( sub, tenantName, - entity.entityId, - entity.types, - entity.payload.asString(), + previousEntity.entityId, + previousEntity.types, + previousEntity.payload.asString(), + serializeObject(deletedEntityPayload.members), emptyList() ) ) @@ -105,27 +105,19 @@ class EntityEventService( suspend fun publishAttributeChangeEvents( sub: String?, entityId: URI, - jsonLdAttributes: Map, - updateResult: UpdateResult, - overwrite: Boolean + attributesOperationsResults: List ): Job { val tenantName = getTenantFromContext() val entity = getSerializedEntity(entityId) return coroutineScope.launch { - logger.debug("Sending attributes change events for entity {} in tenant {}", entityId, tenantName) entity.onRight { - updateResult.updated.forEach { updatedDetails -> - val attributeName = updatedDetails.attributeName - val serializedAttribute = - getSerializedAttribute(jsonLdAttributes, attributeName, updatedDetails.datasetId) + attributesOperationsResults.forEach { attributeOperationResult -> publishAttributeChangeEvent( - updatedDetails, sub, tenantName, entityId, it, - serializedAttribute, - overwrite + attributeOperationResult ) } }.logAttributeEvent("Attribute Change", entityId, tenantName) @@ -133,65 +125,85 @@ class EntityEventService( } private fun publishAttributeChangeEvent( - updatedDetails: UpdatedDetails, sub: String?, tenantName: String, entityId: URI, entityTypesAndPayload: Pair, String>, - serializedAttribute: Pair, - overwrite: Boolean + attributeOperationResult: SucceededAttributeOperationResult ) { - when (updatedDetails.updateOperationResult) { - UpdateOperationResult.APPENDED -> + val attributeName = attributeOperationResult.attributeName + val (types, payload) = entityTypesAndPayload + logger.debug( + "Sending {} event for attribute {} of entity {} in tenant {}", + attributeOperationResult.operationStatus, + attributeName, + entityId, + tenantName + ) + when (attributeOperationResult.operationStatus) { + OperationStatus.APPENDED -> publishEntityEvent( AttributeAppendEvent( sub, tenantName, entityId, - entityTypesAndPayload.first, - serializedAttribute.first, - updatedDetails.datasetId, - overwrite, - serializedAttribute.second, - entityTypesAndPayload.second, + types, + attributeOperationResult.attributeName, + attributeOperationResult.datasetId, + serializeObject(attributeOperationResult.newExpandedValue), + payload, emptyList() ) ) - UpdateOperationResult.REPLACED -> + OperationStatus.REPLACED -> publishEntityEvent( AttributeReplaceEvent( sub, tenantName, entityId, - entityTypesAndPayload.first, - serializedAttribute.first, - updatedDetails.datasetId, - serializedAttribute.second, - entityTypesAndPayload.second, + types, + attributeOperationResult.attributeName, + attributeOperationResult.datasetId, + serializeObject(attributeOperationResult.newExpandedValue), + payload, emptyList() ) ) - UpdateOperationResult.UPDATED -> + OperationStatus.UPDATED -> publishEntityEvent( AttributeUpdateEvent( sub, tenantName, entityId, - entityTypesAndPayload.first, - serializedAttribute.first, - updatedDetails.datasetId, - serializedAttribute.second, - entityTypesAndPayload.second, + types, + attributeOperationResult.attributeName, + attributeOperationResult.datasetId, + serializeObject(attributeOperationResult.newExpandedValue), + payload, + emptyList() + ) + ) + + OperationStatus.DELETED -> + publishEntityEvent( + AttributeDeleteEvent( + sub, + tenantName, + entityId, + types, + attributeOperationResult.attributeName, + attributeOperationResult.datasetId, + injectDeletedAttribute(payload, attributeName, attributeOperationResult.newExpandedValue), emptyList() ) ) else -> logger.warn( - "Received an unexpected result (${updatedDetails.updateOperationResult} " + - "for entity $entityId and attribute ${updatedDetails.attributeName}" + "Received an unexpected result (${attributeOperationResult.operationStatus} " + + "for entity $entityId and attribute ${attributeOperationResult.attributeName}" ) } } @@ -199,11 +211,10 @@ class EntityEventService( suspend fun publishAttributeDeleteEvent( sub: String?, entityId: URI, - attributeName: ExpandedTerm, - datasetId: URI? = null, - deleteAll: Boolean + attributeOperationResult: SucceededAttributeOperationResult ): Job { val tenantName = getTenantFromContext() + val attributeName = attributeOperationResult.attributeName val entity = getSerializedEntity(entityId) return coroutineScope.launch { logger.debug( @@ -213,31 +224,18 @@ class EntityEventService( tenantName ) entity.onRight { - if (deleteAll) - publishEntityEvent( - AttributeDeleteAllInstancesEvent( - sub, - tenantName, - entityId, - it.first, - attributeName, - it.second, - emptyList() - ) - ) - else - publishEntityEvent( - AttributeDeleteEvent( - sub, - tenantName, - entityId, - it.first, - attributeName, - datasetId, - it.second, - emptyList() - ) + publishEntityEvent( + AttributeDeleteEvent( + sub, + tenantName, + entityId, + it.first, + attributeName, + attributeOperationResult.datasetId, + injectDeletedAttribute(it.second, attributeName, attributeOperationResult.newExpandedValue), + emptyList() ) + ) }.logAttributeEvent("Attribute Delete", entityId, tenantName) } } @@ -250,21 +248,19 @@ class EntityEventService( Pair(it.types, it.payload.asString()) } - private fun getSerializedAttribute( - jsonLdAttributes: Map, + internal fun injectDeletedAttribute( + entityPayload: String, attributeName: ExpandedTerm, - datasetId: URI? - ): Pair = - if (attributeName == JSONLD_TYPE) { - Pair(JSONLD_TYPE, serializeObject(jsonLdAttributes[JSONLD_TYPE]!!)) - } else { - val extractedPayload = (jsonLdAttributes as ExpandedAttributes).getAttributeFromExpandedAttributes( - attributeName, - datasetId - )!! - Pair(attributeName, serializeObject(extractedPayload)) + attributeInstance: ExpandedAttributeInstance + ): String { + val entityPayload = entityPayload.deserializeAsMap().toMutableMap() + entityPayload.merge(attributeName, listOf(attributeInstance)) { currentValue, newValue -> + (currentValue as List).plus(newValue as List) } + return serializeObject(entityPayload) + } + private fun Either.logEntityEvent(eventsType: EventsType, entityId: URI, tenantName: String) = this.fold({ logger.error("Error sending {} event for entity {} in tenant {}: {}", eventsType, entityId, tenantName, it) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityQueryService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityQueryService.kt index 7c756b569..782b433df 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityQueryService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityQueryService.kt @@ -18,14 +18,12 @@ import com.egm.stellio.search.entity.model.EntitiesQueryFromPost import com.egm.stellio.search.entity.model.Entity import com.egm.stellio.search.entity.util.rowToEntity import com.egm.stellio.shared.model.APIException -import com.egm.stellio.shared.model.AlreadyExistsException import com.egm.stellio.shared.model.ExpandedEntity import com.egm.stellio.shared.model.ResourceNotFoundException import com.egm.stellio.shared.util.Sub import com.egm.stellio.shared.util.buildQQuery import com.egm.stellio.shared.util.buildScopeQQuery import com.egm.stellio.shared.util.buildTypeQuery -import com.egm.stellio.shared.util.entityAlreadyExistsMessage import com.egm.stellio.shared.util.entityNotFoundMessage import org.springframework.r2dbc.core.DatabaseClient import org.springframework.stereotype.Service @@ -40,11 +38,10 @@ class EntityQueryService( entityId: URI, sub: Sub? = null ): Either = either { - checkEntityExistence(entityId).bind() + val entity = retrieve(entityId).bind() authorizationService.userCanReadEntity(entityId, sub.toOption()).bind() - val entityPayload = retrieve(entityId).bind() - toJsonLdEntity(entityPayload) + toJsonLdEntity(entity) } suspend fun queryEntities( @@ -73,6 +70,13 @@ class EntityQueryService( suspend fun queryEntities( entitiesQuery: EntitiesQuery, accessRightFilter: () -> String? + ): List = + queryEntities(entitiesQuery, true, accessRightFilter) + + suspend fun queryEntities( + entitiesQuery: EntitiesQuery, + excludedDeleted: Boolean = true, + accessRightFilter: () -> String? ): List { val filterQuery = buildFullEntitiesFilter(entitiesQuery, accessRightFilter) @@ -81,8 +85,10 @@ class EntityQueryService( SELECT DISTINCT(entity_payload.entity_id) FROM entity_payload LEFT JOIN temporal_entity_attribute tea - ON tea.entity_id = entity_payload.entity_id + ON tea.entity_id = entity_payload.entity_id + ${if (excludedDeleted) " AND tea.deleted_at is null " else ""} WHERE $filterQuery + ${if (excludedDeleted) " AND entity_payload.deleted_at is null " else ""} ORDER BY entity_id LIMIT :limit OFFSET :offset @@ -98,6 +104,13 @@ class EntityQueryService( suspend fun queryEntitiesCount( entitiesQuery: EntitiesQuery, accessRightFilter: () -> String? + ): Either = + queryEntitiesCount(entitiesQuery, true, accessRightFilter) + + suspend fun queryEntitiesCount( + entitiesQuery: EntitiesQuery, + excludedDeleted: Boolean = true, + accessRightFilter: () -> String? ): Either { val filterQuery = buildFullEntitiesFilter(entitiesQuery, accessRightFilter) @@ -107,7 +120,9 @@ class EntityQueryService( FROM entity_payload LEFT JOIN temporal_entity_attribute tea ON tea.entity_id = entity_payload.entity_id + ${if (excludedDeleted) " AND tea.deleted_at is null " else ""} WHERE $filterQuery + ${if (excludedDeleted) " AND entity_payload.deleted_at is null " else ""} """.trimIndent() return databaseClient @@ -212,15 +227,16 @@ class EntityQueryService( return entitySelectorFilter?.joinToString(separator = " OR ") ?: " 1 = 1 " } - suspend fun retrieve(entityId: URI): Either = + suspend fun retrieve(entityId: URI, allowDeleted: Boolean = false): Either = databaseClient.sql( """ SELECT * from entity_payload WHERE entity_id = :entity_id + ${if (!allowDeleted) " and deleted_at is null " else ""} """.trimIndent() ) .bind("entity_id", entityId) - .oneToResult { it.rowToEntity() } + .oneToResult(ResourceNotFoundException(entityNotFoundMessage(entityId.toString()))) { it.rowToEntity() } suspend fun retrieve(entitiesIds: List): List = databaseClient.sql( @@ -232,10 +248,7 @@ class EntityQueryService( .bind("entities_ids", entitiesIds) .allToMappedList { it.rowToEntity() } - suspend fun checkEntityExistence( - entityId: URI, - inverse: Boolean = false - ): Either { + suspend fun checkEntityExistence(entityId: URI, allowDeleted: Boolean = false): Either { val selectQuery = """ select @@ -243,6 +256,7 @@ class EntityQueryService( select 1 from entity_payload where entity_id = :entity_id + ${if (!allowDeleted) " and deleted_at is null " else ""} ) as entityExists; """.trimIndent() @@ -251,15 +265,32 @@ class EntityQueryService( .bind("entity_id", entityId) .oneToResult { it["entityExists"] as Boolean } .flatMap { - if (it && !inverse || !it && inverse) + if (it) Unit.right() - else if (it) - AlreadyExistsException(entityAlreadyExistsMessage(entityId.toString())).left() else ResourceNotFoundException(entityNotFoundMessage(entityId.toString())).left() } } + /** + * Used for checks before creating a (temporal) entity. Allows to know if the entity does not exist, + * or, if it exists, whether it is currently deleted (in which case, it may be possible to create it again + * if authorized) + */ + suspend fun isMarkedAsDeleted(entityId: URI): Either { + val selectQuery = + """ + select entity_id, deleted_at + from entity_payload + where entity_id = :entity_id + """.trimIndent() + + return databaseClient + .sql(selectQuery) + .bind("entity_id", entityId) + .oneToResult { it["deleted_at"] != null } + } + suspend fun filterExistingEntitiesAsIds(entitiesIds: List): List { if (entitiesIds.isEmpty()) { return emptyList() @@ -270,6 +301,7 @@ class EntityQueryService( select entity_id from entity_payload where entity_id in (:entities_ids) + and deleted_at is null """.trimIndent() return databaseClient diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityService.kt index 800d741a5..0f5acd87b 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/service/EntityService.kt @@ -1,6 +1,9 @@ package com.egm.stellio.search.entity.service import arrow.core.Either +import arrow.core.Either.Left +import arrow.core.Either.Right +import arrow.core.left import arrow.core.raise.either import arrow.core.right import arrow.core.toOption @@ -10,21 +13,25 @@ import com.egm.stellio.search.common.util.execute import com.egm.stellio.search.common.util.oneToResult import com.egm.stellio.search.common.util.toZonedDateTime import com.egm.stellio.search.entity.model.Attribute -import com.egm.stellio.search.entity.model.EMPTY_UPDATE_RESULT +import com.egm.stellio.search.entity.model.AttributeOperationResult import com.egm.stellio.search.entity.model.Entity +import com.egm.stellio.search.entity.model.FailedAttributeOperationResult +import com.egm.stellio.search.entity.model.OperationStatus import com.egm.stellio.search.entity.model.OperationType import com.egm.stellio.search.entity.model.OperationType.APPEND_ATTRIBUTES import com.egm.stellio.search.entity.model.OperationType.APPEND_ATTRIBUTES_OVERWRITE_ALLOWED import com.egm.stellio.search.entity.model.OperationType.MERGE_ENTITY import com.egm.stellio.search.entity.model.OperationType.UPDATE_ATTRIBUTES -import com.egm.stellio.search.entity.model.UpdateAttributeResult -import com.egm.stellio.search.entity.model.UpdateOperationResult +import com.egm.stellio.search.entity.model.SucceededAttributeOperationResult import com.egm.stellio.search.entity.model.UpdateResult -import com.egm.stellio.search.entity.model.updateResultFromDetailedResult +import com.egm.stellio.search.entity.model.getSucceededOperations +import com.egm.stellio.search.entity.model.hasSuccessfulResult import com.egm.stellio.search.entity.util.prepareAttributes import com.egm.stellio.search.entity.util.rowToEntity import com.egm.stellio.search.scope.ScopeService +import com.egm.stellio.search.temporal.model.AttributeInstance.TemporalProperty import com.egm.stellio.shared.model.APIException +import com.egm.stellio.shared.model.AlreadyExistsException import com.egm.stellio.shared.model.ExpandedAttribute import com.egm.stellio.shared.model.ExpandedAttributeInstances import com.egm.stellio.shared.model.ExpandedAttributes @@ -32,13 +39,14 @@ import com.egm.stellio.shared.model.ExpandedEntity import com.egm.stellio.shared.model.ExpandedTerm import com.egm.stellio.shared.model.NgsiLdEntity import com.egm.stellio.shared.model.addSysAttrs -import com.egm.stellio.shared.model.toExpandedAttributes import com.egm.stellio.shared.model.toNgsiLdAttributes import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_EXPANDED_ENTITY_SPECIFIC_MEMBERS +import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_ID import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_SCOPE_PROPERTY import com.egm.stellio.shared.util.JsonUtils.serializeObject import com.egm.stellio.shared.util.Sub +import com.egm.stellio.shared.util.entityAlreadyExistsMessage import com.egm.stellio.shared.util.getSpecificAccessPolicy import com.egm.stellio.shared.util.ngsiLdDateTime import io.r2dbc.postgresql.codec.Json @@ -48,7 +56,6 @@ import org.springframework.r2dbc.core.bind import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional import java.net.URI -import java.time.ZoneOffset import java.time.ZonedDateTime @Service @@ -68,14 +75,23 @@ class EntityService( expandedEntity: ExpandedEntity, sub: Sub? = null ): Either = either { - authorizationService.userCanCreateEntities(sub.toOption()).bind() - entityQueryService.checkEntityExistence(ngsiLdEntity.id, true).bind() + entityQueryService.isMarkedAsDeleted(ngsiLdEntity.id).let { + when (it) { + is Left -> authorizationService.userCanCreateEntities(sub.toOption()).bind() + is Right -> + if (!it.value) + AlreadyExistsException(entityAlreadyExistsMessage(ngsiLdEntity.id.toString())).left().bind() + else + authorizationService.userCanAdminEntity(ngsiLdEntity.id, sub.toOption()).bind() + } + } val createdAt = ngsiLdDateTime() val attributesMetadata = ngsiLdEntity.prepareAttributes().bind() logger.debug("Creating entity {}", ngsiLdEntity.id) - createEntityPayload(ngsiLdEntity, expandedEntity, createdAt, sub).bind() + createEntityPayload(ngsiLdEntity, expandedEntity, createdAt).bind() + scopeService.createHistory(ngsiLdEntity, createdAt, sub).bind() entityAttributeService.createAttributes( ngsiLdEntity, expandedEntity, @@ -96,14 +112,20 @@ class EntityService( suspend fun createEntityPayload( ngsiLdEntity: NgsiLdEntity, expandedEntity: ExpandedEntity, - createdAt: ZonedDateTime, - sub: Sub? = null + createdAt: ZonedDateTime ): Either = either { val specificAccessPolicy = ngsiLdEntity.getSpecificAccessPolicy()?.bind() databaseClient.sql( """ INSERT INTO entity_payload (entity_id, types, scopes, created_at, payload, specific_access_policy) VALUES (:entity_id, :types, :scopes, :created_at, :payload, :specific_access_policy) + ON CONFLICT (entity_id) + DO UPDATE SET types = :types, + scopes = :scopes, + modified_at = :created_at, + deleted_at = null, + payload = :payload, + specific_access_policy = :specific_access_policy """.trimIndent() ) .bind("entity_id", ngsiLdEntity.id) @@ -113,9 +135,6 @@ class EntityService( .bind("payload", Json.of(serializeObject(expandedEntity.populateCreationTimeDate(createdAt).members))) .bind("specific_access_policy", specificAccessPolicy?.toString()) .execute() - .map { - scopeService.createHistory(ngsiLdEntity, createdAt, sub) - } } @Transactional @@ -129,12 +148,15 @@ class EntityService( authorizationService.userCanUpdateEntity(entityId, sub.toOption()).bind() val (coreAttrs, otherAttrs) = - expandedAttributes.toList().partition { JSONLD_EXPANDED_ENTITY_SPECIFIC_MEMBERS.contains(it.first) } + expandedAttributes.toList() + // remove @id if it is present (optional as per 5.4) + .filter { it.first != JSONLD_ID } + .partition { JSONLD_EXPANDED_ENTITY_SPECIFIC_MEMBERS.contains(it.first) } val mergedAt = ngsiLdDateTime() logger.debug("Merging entity {}", entityId) - val coreUpdateResult = updateCoreAttributes(entityId, coreAttrs, mergedAt, MERGE_ENTITY).bind() - val attrsUpdateResult = entityAttributeService.mergeAttributes( + val coreOperationResult = updateCoreAttributes(entityId, coreAttrs, mergedAt, MERGE_ENTITY).bind() + val attrsOperationResult = entityAttributeService.mergeAttributes( entityId, otherAttrs.toMap().toNgsiLdAttributes().bind(), expandedAttributes, @@ -143,24 +165,20 @@ class EntityService( sub ).bind() - val updateResult = coreUpdateResult.mergeWith(attrsUpdateResult) + val operationResult = coreOperationResult.plus(attrsOperationResult) // update modifiedAt in entity if at least one attribute has been merged - if (updateResult.hasSuccessfulUpdate()) { + if (operationResult.hasSuccessfulResult()) { val attributes = entityAttributeService.getForEntity(entityId, emptySet(), emptySet()) updateState(entityId, mergedAt, attributes).bind() - } - if (updateResult.updated.isNotEmpty()) { entityEventService.publishAttributeChangeEvents( sub, entityId, - expandedAttributes, - updateResult, - true + operationResult.getSucceededOperations() ) } - updateResult + UpdateResult(operationResult) } @Transactional @@ -173,13 +191,14 @@ class EntityService( entityQueryService.checkEntityExistence(entityId).bind() authorizationService.userCanUpdateEntity(entityId, sub.toOption()).bind() - val replacedAt = ngsiLdDateTime() val attributesMetadata = ngsiLdEntity.prepareAttributes().bind() logger.debug("Replacing entity {}", ngsiLdEntity.id) - entityAttributeService.deleteAttributes(entityId) + entityAttributeService.deleteAttributes(entityId, ngsiLdDateTime()).bind() - replaceEntityPayload(ngsiLdEntity, expandedEntity, replacedAt, sub).bind() + val replacedAt = ngsiLdDateTime() + replaceEntityPayload(ngsiLdEntity, expandedEntity, replacedAt).bind() + scopeService.replace(ngsiLdEntity, replacedAt, sub).bind() entityAttributeService.createAttributes( ngsiLdEntity, expandedEntity, @@ -199,8 +218,7 @@ class EntityService( suspend fun replaceEntityPayload( ngsiLdEntity: NgsiLdEntity, expandedEntity: ExpandedEntity, - replacedAt: ZonedDateTime, - sub: Sub? = null + replacedAt: ZonedDateTime ): Either = either { val specificAccessPolicy = ngsiLdEntity.getSpecificAccessPolicy()?.bind() val createdAt = retrieveCreatedAt(ngsiLdEntity.id).bind() @@ -225,9 +243,6 @@ class EntityService( .bind("payload", Json.of(serializedPayload)) .bind("specific_access_policy", specificAccessPolicy?.toString()) .execute() - .map { - scopeService.replaceHistoryEntry(ngsiLdEntity, createdAt, sub) - } } private suspend fun retrieveCreatedAt(entityId: URI): Either = @@ -246,7 +261,7 @@ class EntityService( coreAttrs: List>, modifiedAt: ZonedDateTime, operationType: OperationType - ): Either = either { + ): Either> = either { coreAttrs.map { (expandedTerm, expandedAttributeInstances) -> when (expandedTerm) { JSONLD_TYPE -> @@ -255,11 +270,14 @@ class EntityService( scopeService.update(entityId, expandedAttributeInstances, modifiedAt, operationType).bind() else -> { logger.warn("Ignoring unhandled core property: {}", expandedTerm) - EMPTY_UPDATE_RESULT.right().bind() + FailedAttributeOperationResult( + attributeName = expandedTerm, + operationStatus = OperationStatus.IGNORED, + errorMessage = "Ignoring unhandled core property: $expandedTerm" + ).right().bind() } } - }.ifEmpty { listOf(EMPTY_UPDATE_RESULT) } - .reduce { acc, cur -> acc.mergeWith(cur) } + } } @Transactional @@ -268,12 +286,16 @@ class EntityService( newTypes: List, modifiedAt: ZonedDateTime, allowEmptyListOfTypes: Boolean = true - ): Either = either { + ): Either = either { val entityPayload = entityQueryService.retrieve(entityId).bind() val currentTypes = entityPayload.types // when dealing with an entity update, list of types can be empty if no change of type is requested if (currentTypes.sorted() == newTypes.sorted() || newTypes.isEmpty() && allowEmptyListOfTypes) - return@either UpdateResult(emptyList(), emptyList()) + return@either SucceededAttributeOperationResult( + attributeName = JSONLD_TYPE, + operationStatus = OperationStatus.APPENDED, + newExpandedValue = mapOf(JSONLD_TYPE to currentTypes.toList()) + ) val updatedTypes = currentTypes.union(newTypes) val updatedPayload = entityPayload.payload.deserializeExpandedPayload() @@ -298,13 +320,10 @@ class EntityService( .bind("payload", Json.of(serializeObject(updatedPayload))) .execute() .map { - updateResultFromDetailedResult( - listOf( - UpdateAttributeResult( - attributeName = JSONLD_TYPE, - updateOperationResult = UpdateOperationResult.APPENDED - ) - ) + SucceededAttributeOperationResult( + attributeName = JSONLD_TYPE, + operationStatus = OperationStatus.APPENDED, + newExpandedValue = mapOf(JSONLD_TYPE to updatedTypes.toList()) ) }.bind() } @@ -326,8 +345,8 @@ class EntityService( val operationType = if (disallowOverwrite) APPEND_ATTRIBUTES else APPEND_ATTRIBUTES_OVERWRITE_ALLOWED - val coreUpdateResult = updateCoreAttributes(entityId, coreAttrs, createdAt, operationType).bind() - val attrsUpdateResult = entityAttributeService.appendAttributes( + val coreOperationResult = updateCoreAttributes(entityId, coreAttrs, createdAt, operationType).bind() + val attrsOperationResult = entityAttributeService.appendAttributes( entityId, otherAttrs.toMap().toNgsiLdAttributes().bind(), expandedAttributes, @@ -336,24 +355,10 @@ class EntityService( sub ).bind() - val updateResult = coreUpdateResult.mergeWith(attrsUpdateResult) - // update modifiedAt in entity if at least one attribute has been added - if (updateResult.hasSuccessfulUpdate()) { - val attributes = entityAttributeService.getForEntity(entityId, emptySet(), emptySet()) - updateState(entityId, createdAt, attributes).bind() - } - - if (updateResult.hasSuccessfulUpdate()) { - entityEventService.publishAttributeChangeEvents( - sub, - entityId, - expandedAttributes, - updateResult, - true - ) - } + val operationResult = coreOperationResult.plus(attrsOperationResult) + handleSuccessOperationActions(operationResult, entityId, createdAt, sub).bind() - updateResult + UpdateResult(operationResult) } @Transactional @@ -369,8 +374,8 @@ class EntityService( expandedAttributes.toList().partition { JSONLD_EXPANDED_ENTITY_SPECIFIC_MEMBERS.contains(it.first) } val createdAt = ngsiLdDateTime() - val coreUpdateResult = updateCoreAttributes(entityId, coreAttrs, createdAt, UPDATE_ATTRIBUTES).bind() - val attrsUpdateResult = entityAttributeService.updateAttributes( + val coreOperationResult = updateCoreAttributes(entityId, coreAttrs, createdAt, UPDATE_ATTRIBUTES).bind() + val attrsOperationResult = entityAttributeService.updateAttributes( entityId, otherAttrs.toMap().toNgsiLdAttributes().bind(), expandedAttributes, @@ -378,24 +383,10 @@ class EntityService( sub ).bind() - val updateResult = coreUpdateResult.mergeWith(attrsUpdateResult) - // update modifiedAt in entity if at least one attribute has been added - if (updateResult.hasSuccessfulUpdate()) { - val attributes = entityAttributeService.getForEntity(entityId, emptySet(), emptySet()) - updateState(entityId, createdAt, attributes).bind() - } - - if (updateResult.updated.isNotEmpty()) { - entityEventService.publishAttributeChangeEvents( - sub, - entityId, - expandedAttributes, - updateResult, - true - ) - } + val operationResult = coreOperationResult.plus(attrsOperationResult) + handleSuccessOperationActions(operationResult, entityId, createdAt, sub).bind() - updateResult + UpdateResult(operationResult) } @Transactional @@ -409,28 +400,16 @@ class EntityService( val modifiedAt = ngsiLdDateTime() - val updateResult = entityAttributeService.partialUpdateAttribute( + val operationResult = entityAttributeService.partialUpdateAttribute( entityId, expandedAttribute, modifiedAt, sub - ).bind() + ).bind().let { listOf(it) } - if (updateResult.isSuccessful()) { - val attributes = entityAttributeService.getForEntity(entityId, emptySet(), emptySet()) - updateState(entityId, modifiedAt, attributes).bind() - } - - if (updateResult.updated.isNotEmpty()) - entityEventService.publishAttributeChangeEvents( - sub, - entityId, - expandedAttribute.toExpandedAttributes(), - updateResult, - false - ) + handleSuccessOperationActions(operationResult, entityId, modifiedAt, sub).bind() - updateResult + UpdateResult(operationResult) } @Transactional @@ -439,7 +418,7 @@ class EntityService( expandedAttributes: ExpandedAttributes, sub: Sub? = null ): Either = either { - val createdAt = ZonedDateTime.now(ZoneOffset.UTC) + val createdAt = ngsiLdDateTime() expandedAttributes.forEach { (attributeName, expandedAttributeInstances) -> expandedAttributeInstances.forEach { expandedAttributeInstance -> val jsonLdAttribute = mapOf(attributeName to listOf(expandedAttributeInstance)) @@ -473,30 +452,37 @@ class EntityService( val ngsiLdAttribute = listOf(expandedAttribute).toMap().toNgsiLdAttributes().bind()[0] val replacedAt = ngsiLdDateTime() - val updateResult = entityAttributeService.replaceAttribute( + val operationResult = entityAttributeService.replaceAttribute( entityId, ngsiLdAttribute, expandedAttribute, replacedAt, sub - ).bind() + ).bind().let { listOf(it) } + + handleSuccessOperationActions(operationResult, entityId, replacedAt, sub).bind() + + UpdateResult(operationResult) + } + @Transactional + internal suspend fun handleSuccessOperationActions( + operationResult: List, + entityId: URI, + createdAt: ZonedDateTime, + sub: Sub? = null + ): Either = either { // update modifiedAt in entity if at least one attribute has been added - if (updateResult.hasSuccessfulUpdate()) { + if (operationResult.hasSuccessfulResult()) { val attributes = entityAttributeService.getForEntity(entityId, emptySet(), emptySet()) - updateState(entityId, replacedAt, attributes).bind() - } + updateState(entityId, createdAt, attributes).bind() - if (updateResult.updated.isNotEmpty()) entityEventService.publishAttributeChangeEvents( sub, entityId, - expandedAttribute.toExpandedAttributes(), - updateResult, - false + operationResult.getSucceededOperations() ) - - updateResult + } } @Transactional @@ -545,36 +531,72 @@ class EntityService( } @Transactional - suspend fun upsertEntityPayload(entityId: URI, payload: String): Either = + suspend fun deleteEntity(entityId: URI, sub: Sub? = null): Either = either { + val currentEntity = entityQueryService.retrieve(entityId).bind() + authorizationService.userCanAdminEntity(entityId, sub.toOption()).bind() + + val deletedAt = ngsiLdDateTime() + val deletedEntityPayload = currentEntity.toExpandedDeletedEntity(deletedAt) + val previousEntity = deleteEntityPayload(entityId, deletedAt, deletedEntityPayload).bind() + entityAttributeService.deleteAttributes(entityId, deletedAt).bind() + scopeService.addHistoryEntry(entityId, emptyList(), TemporalProperty.DELETED_AT, deletedAt, sub).bind() + + entityEventService.publishEntityDeleteEvent(sub, previousEntity, deletedEntityPayload) + } + + @Transactional + suspend fun deleteEntityPayload( + entityId: URI, + deletedAt: ZonedDateTime, + deletedEntityPayload: ExpandedEntity + ): Either = either { databaseClient.sql( """ - INSERT INTO entity_payload (entity_id, payload) - VALUES (:entity_id, :payload) - ON CONFLICT (entity_id) - DO UPDATE SET payload = :payload + WITH entity_before_delete AS ( + SELECT * + FROM entity_payload + WHERE entity_id = :entity_id + ), + update_entity AS ( + UPDATE entity_payload + SET deleted_at = :deleted_at, + payload = :payload, + scopes = null, + specific_access_policy = null, + types = '{}' + WHERE entity_id = :entity_id + ) + SELECT * FROM entity_before_delete """.trimIndent() ) - .bind("payload", Json.of(payload)) .bind("entity_id", entityId) - .execute() + .bind("deleted_at", deletedAt) + .bind("payload", Json.of(serializeObject(deletedEntityPayload.members))) + .oneToResult { + it.rowToEntity() + } + .bind() + } @Transactional - suspend fun deleteEntity(entityId: URI, sub: Sub? = null): Either = either { - entityQueryService.checkEntityExistence(entityId).bind() + suspend fun permanentlyDeleteEntity(entityId: URI, sub: Sub? = null): Either = either { + val currentEntity = entityQueryService.retrieve(entityId, true).bind() authorizationService.userCanAdminEntity(entityId, sub.toOption()).bind() - val entity = deleteEntityPayload(entityId).bind() - - entityAttributeService.deleteAttributes(entityId).bind() - scopeService.deleteHistory(entityId).bind() + val previousEntity = permanentyDeleteEntityPayload(entityId).bind() + entityAttributeService.permanentlyDeleteAttributes(entityId).bind() authorizationService.removeRightsOnEntity(entityId).bind() - entityEventService.publishEntityDeleteEvent(sub, entity) + if (currentEntity.deletedAt == null) { + // only send a notification if entity was not already previously deleted + val deletedEntityPayload = currentEntity.toExpandedDeletedEntity(ngsiLdDateTime()) + entityEventService.publishEntityDeleteEvent(sub, previousEntity, deletedEntityPayload) + } } @Transactional - suspend fun deleteEntityPayload(entityId: URI): Either = either { - val entity = databaseClient.sql( + suspend fun permanentyDeleteEntityPayload(entityId: URI): Either = either { + databaseClient.sql( """ DELETE FROM entity_payload WHERE entity_id = :entity_id @@ -586,7 +608,6 @@ class EntityService( it.rowToEntity() } .bind() - entity } @Transactional @@ -599,7 +620,7 @@ class EntityService( ): Either = either { authorizationService.userCanUpdateEntity(entityId, sub.toOption()).bind() - if (attributeName == NGSILD_SCOPE_PROPERTY) { + val deleteAttributeResults = if (attributeName == NGSILD_SCOPE_PROPERTY) { scopeService.delete(entityId).bind() } else { entityAttributeService.checkEntityAndAttributeExistence( @@ -612,7 +633,8 @@ class EntityService( entityId, attributeName, datasetId, - deleteAll + deleteAll, + ngsiLdDateTime() ).bind() } updateState( @@ -621,12 +643,41 @@ class EntityService( entityAttributeService.getForEntity(entityId, emptySet(), emptySet()) ).bind() - entityEventService.publishAttributeDeleteEvent( - sub, + deleteAttributeResults.filterIsInstance() + .forEach { + entityEventService.publishAttributeDeleteEvent(sub, entityId, it) + } + } + + @Transactional + suspend fun permanentlyDeleteAttribute( + entityId: URI, + attributeName: ExpandedTerm, + datasetId: URI?, + deleteAll: Boolean = false, + sub: Sub? = null + ): Either = either { + authorizationService.userCanUpdateEntity(entityId, sub.toOption()).bind() + + if (attributeName == NGSILD_SCOPE_PROPERTY) { + scopeService.permanentlyDelete(entityId).bind() + } else { + entityAttributeService.checkEntityAndAttributeExistence( + entityId, + attributeName, + datasetId + ).bind() + entityAttributeService.permanentlyDeleteAttribute( + entityId, + attributeName, + datasetId, + deleteAll + ).bind() + } + updateState( entityId, - attributeName, - datasetId, - deleteAll - ) + ngsiLdDateTime(), + entityAttributeService.getForEntity(entityId, emptySet(), emptySet()) + ).bind() } } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/util/AttributeUtils.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/util/AttributeUtils.kt index 32d2c2c35..af8e883d6 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/util/AttributeUtils.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/util/AttributeUtils.kt @@ -7,6 +7,7 @@ import arrow.core.right import com.egm.stellio.search.common.util.deserializeAsMap import com.egm.stellio.search.common.util.valueToDoubleOrNull import com.egm.stellio.search.entity.model.Attribute +import com.egm.stellio.search.entity.model.Attribute.AttributeType import com.egm.stellio.search.entity.model.AttributeMetadata import com.egm.stellio.shared.model.APIException import com.egm.stellio.shared.model.BadRequestDataException @@ -20,12 +21,17 @@ import com.egm.stellio.shared.model.NgsiLdPropertyInstance import com.egm.stellio.shared.model.NgsiLdRelationshipInstance import com.egm.stellio.shared.model.NgsiLdVocabPropertyInstance import com.egm.stellio.shared.model.WKTCoordinates +import com.egm.stellio.shared.model.getMemberValue import com.egm.stellio.shared.model.getPropertyValue +import com.egm.stellio.shared.model.getRelationshipId import com.egm.stellio.shared.util.JsonLdUtils +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_LANGUAGEPROPERTY_VALUE +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_NULL import com.egm.stellio.shared.util.JsonUtils import com.egm.stellio.shared.util.JsonUtils.deserializeAsMap import com.savvasdalkitsis.jsonmerger.JsonMerger import io.r2dbc.postgresql.codec.Json +import java.net.URI import java.time.LocalDate import java.time.LocalTime import java.time.ZonedDateTime @@ -47,35 +53,35 @@ fun NgsiLdAttributeInstance.toAttributeMetadata(): Either guessPropertyValueType(this).let { - Triple(Attribute.AttributeType.Property, it.first, it.second) + Triple(AttributeType.Property, it.first, it.second) } is NgsiLdRelationshipInstance -> Triple( - Attribute.AttributeType.Relationship, + AttributeType.Relationship, Attribute.AttributeValueType.URI, Triple(this.objectId.toString(), null, null) ) is NgsiLdGeoPropertyInstance -> Triple( - Attribute.AttributeType.GeoProperty, + AttributeType.GeoProperty, Attribute.AttributeValueType.GEOMETRY, Triple(null, null, this.coordinates) ) is NgsiLdJsonPropertyInstance -> Triple( - Attribute.AttributeType.JsonProperty, + AttributeType.JsonProperty, Attribute.AttributeValueType.JSON, Triple(JsonUtils.serializeObject(this.json), null, null) ) is NgsiLdLanguagePropertyInstance -> Triple( - Attribute.AttributeType.LanguageProperty, + AttributeType.LanguageProperty, Attribute.AttributeValueType.ARRAY, Triple(JsonUtils.serializeObject(this.languageMap), null, null) ) is NgsiLdVocabPropertyInstance -> Triple( - Attribute.AttributeType.VocabProperty, + AttributeType.VocabProperty, Attribute.AttributeValueType.ARRAY, Triple(JsonUtils.serializeObject(this.vocab), null, null) ) @@ -97,17 +103,17 @@ fun NgsiLdAttributeInstance.toAttributeMetadata(): Either + AttributeType.Property -> guessPropertyValueType(expandedAttributeInstance.getPropertyValue()!!).first - Attribute.AttributeType.Relationship -> Attribute.AttributeValueType.URI - Attribute.AttributeType.GeoProperty -> Attribute.AttributeValueType.GEOMETRY - Attribute.AttributeType.JsonProperty -> Attribute.AttributeValueType.JSON - Attribute.AttributeType.LanguageProperty -> Attribute.AttributeValueType.ARRAY - Attribute.AttributeType.VocabProperty -> Attribute.AttributeValueType.ARRAY + AttributeType.Relationship -> Attribute.AttributeValueType.URI + AttributeType.GeoProperty -> Attribute.AttributeValueType.GEOMETRY + AttributeType.JsonProperty -> Attribute.AttributeValueType.JSON + AttributeType.LanguageProperty -> Attribute.AttributeValueType.ARRAY + AttributeType.VocabProperty -> Attribute.AttributeValueType.ARRAY } fun guessPropertyValueType( @@ -131,6 +137,21 @@ fun guessPropertyValueType( else -> Pair(Attribute.AttributeValueType.STRING, Triple(value.toString(), null, null)) } +/** + * Returns whether the expanded attribute instance holds a NGSI-LD Null value + */ +fun hasNgsiLdNullValue( + expandedAttributeInstance: ExpandedAttributeInstance, + attributeType: AttributeType +): Boolean = + if (attributeType == AttributeType.Relationship) { + val value = expandedAttributeInstance.getRelationshipId() + value is URI && value.toString() == NGSILD_NULL + } else { + val value = expandedAttributeInstance.getMemberValue(attributeType.toExpandedValueMember()) + value is String && value == NGSILD_NULL + } + fun Json.toExpandedAttributeInstance(): ExpandedAttributeInstance = this.deserializeAsMap() as ExpandedAttributeInstance @@ -169,7 +190,7 @@ fun mergePatch( ).deserializeAsMap() ) } - } else if (listOf(JsonLdUtils.NGSILD_LANGUAGEPROPERTY_VALUE).contains(attrName)) { + } else if (listOf(NGSILD_LANGUAGEPROPERTY_VALUE).contains(attrName)) { val sourceLangEntries = source[attrName] as List> val targetLangEntries = sourceLangEntries.toMutableList() (attrValue as List>).forEach { langEntry -> diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/entity/util/EntityUtils.kt b/search-service/src/main/kotlin/com/egm/stellio/search/entity/util/EntityUtils.kt index 09af4abf5..7e5c40b05 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/entity/util/EntityUtils.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/entity/util/EntityUtils.kt @@ -17,6 +17,7 @@ fun Map.rowToEntity(): Entity = scopes = toOptionalList(this["scopes"]), createdAt = toZonedDateTime(this["created_at"]), modifiedAt = toOptionalZonedDateTime(this["modified_at"]), + deletedAt = toOptionalZonedDateTime(this["deleted_at"]), payload = toJson(this["payload"]), specificAccessPolicy = toOptionalEnum(this["specific_access_policy"]) ) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt index 2968f7cc6..9eacfef87 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/scope/ScopeService.kt @@ -16,12 +16,11 @@ import com.egm.stellio.search.common.util.toOptionalZonedDateTime import com.egm.stellio.search.common.util.toUri import com.egm.stellio.search.common.util.toZonedDateTime import com.egm.stellio.search.entity.model.Attribute.AttributeValueType -import com.egm.stellio.search.entity.model.NotUpdatedDetails +import com.egm.stellio.search.entity.model.AttributeOperationResult +import com.egm.stellio.search.entity.model.FailedAttributeOperationResult +import com.egm.stellio.search.entity.model.OperationStatus import com.egm.stellio.search.entity.model.OperationType -import com.egm.stellio.search.entity.model.UpdateAttributeResult -import com.egm.stellio.search.entity.model.UpdateOperationResult -import com.egm.stellio.search.entity.model.UpdateResult -import com.egm.stellio.search.entity.model.updateResultFromDetailedResult +import com.egm.stellio.search.entity.model.SucceededAttributeOperationResult import com.egm.stellio.search.temporal.model.AttributeInstance.TemporalProperty import com.egm.stellio.search.temporal.model.TemporalEntitiesQuery import com.egm.stellio.search.temporal.model.TemporalQuery @@ -37,6 +36,8 @@ import com.egm.stellio.shared.util.JsonLdUtils import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_SCOPE_PROPERTY import com.egm.stellio.shared.util.JsonUtils.serializeObject import com.egm.stellio.shared.util.Sub +import com.egm.stellio.shared.util.getSubFromSecurityContext +import com.egm.stellio.shared.util.ngsiLdDateTime import io.r2dbc.postgresql.codec.Json import org.springframework.r2dbc.core.DatabaseClient import org.springframework.r2dbc.core.bind @@ -72,7 +73,7 @@ class ScopeService( suspend fun addHistoryEntry( entityId: URI, scopes: List, - temportalProperty: TemporalProperty, + temporalProperty: TemporalProperty, createdAt: ZonedDateTime, sub: Sub? = null ): Either = @@ -85,7 +86,7 @@ class ScopeService( .bind("entity_id", entityId) .bind("value", scopes.toTypedArray()) .bind("time", createdAt) - .bind("time_property", temportalProperty.toString()) + .bind("time_property", temporalProperty.toString()) .bind("sub", sub) .execute() @@ -253,7 +254,7 @@ class ScopeService( modifiedAt: ZonedDateTime, operationType: OperationType, sub: Sub? = null - ): Either = either { + ): Either = either { val scopes = mapOf(NGSILD_SCOPE_PROPERTY to expandedAttributeInstances).getScopes()!! val (currentScopes, currentPayload) = retrieve(entityId).bind() @@ -262,14 +263,10 @@ class ScopeService( if (currentScopes != null) { val updatedPayload = currentPayload.replaceScopeValue(expandedAttributeInstances) Pair(scopes, updatedPayload) - } else return@either UpdateResult( - updated = emptyList(), - notUpdated = listOf( - NotUpdatedDetails( - NGSILD_SCOPE_PROPERTY, - "Attribute does not exist and operation does not allow creating it" - ) - ) + } else return@either FailedAttributeOperationResult( + attributeName = NGSILD_SCOPE_PROPERTY, + operationStatus = OperationStatus.FAILED, + errorMessage = "Scope does not exist and operation does not allow creating it" ) } OperationType.APPEND_ATTRIBUTES, OperationType.MERGE_ENTITY -> { @@ -289,7 +286,7 @@ class ScopeService( } updatedScopes?.let { - val updateResult = + val operationResult = performUpdate(entityId, updatedScopes, modifiedAt, serializeObject(updatedPayload)).bind() val temporalPropertyToAdd = if (currentScopes == null) TemporalProperty.CREATED_AT @@ -300,10 +297,11 @@ class ScopeService( // change from the Core API, the observedAt sub-Property should be set as a copy of the modifiedAt // sub-Property addHistoryEntry(entityId, it, TemporalProperty.OBSERVED_AT, modifiedAt, sub).bind() - updateResult - } ?: UpdateResult( - emptyList(), - listOf(NotUpdatedDetails(NGSILD_SCOPE_PROPERTY, "Unrecognized operation type: $operationType")) + operationResult + } ?: FailedAttributeOperationResult( + attributeName = NGSILD_SCOPE_PROPERTY, + operationStatus = OperationStatus.FAILED, + errorMessage = "Unrecognized operation type on scope: $operationType" ) } @@ -313,7 +311,7 @@ class ScopeService( scopes: List, modifiedAt: ZonedDateTime, payload: String - ): Either = either { + ): Either = either { databaseClient.sql( """ UPDATE entity_payload @@ -329,29 +327,25 @@ class ScopeService( .bind("payload", Json.of(payload)) .execute() .map { - updateResultFromDetailedResult( - listOf( - UpdateAttributeResult( - attributeName = NGSILD_SCOPE_PROPERTY, - updateOperationResult = UpdateOperationResult.APPENDED - ) - ) + SucceededAttributeOperationResult( + attributeName = NGSILD_SCOPE_PROPERTY, + operationStatus = OperationStatus.APPENDED, + newExpandedValue = mapOf(NGSILD_SCOPE_PROPERTY to scopes.toList()) ) }.bind() } @Transactional - suspend fun replaceHistoryEntry( + suspend fun replace( ngsiLdEntity: NgsiLdEntity, createdAt: ZonedDateTime, sub: Sub? = null ): Either = either { - deleteHistory(ngsiLdEntity.id).bind() createHistory(ngsiLdEntity, createdAt, sub).bind() } @Transactional - suspend fun delete(entityId: URI): Either = either { + suspend fun delete(entityId: URI): Either> = either { databaseClient.sql( """ UPDATE entity_payload @@ -364,10 +358,26 @@ class ScopeService( .execute() .bind() - deleteHistory(entityId).bind() + addHistoryEntry( + entityId, + emptyList(), + TemporalProperty.DELETED_AT, + ngsiLdDateTime(), + getSubFromSecurityContext().getOrNull() + ).bind() + + listOf( + SucceededAttributeOperationResult( + NGSILD_SCOPE_PROPERTY, + null, + OperationStatus.DELETED, + mapOf(NGSILD_SCOPE_PROPERTY to listOf()) + ) + ) } - suspend fun deleteHistory(entityId: URI): Either = + @Transactional + suspend fun permanentlyDelete(entityId: URI): Either = databaseClient.sql( """ DELETE FROM scope_history diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/temporal/model/AttributeInstance.kt b/search-service/src/main/kotlin/com/egm/stellio/search/temporal/model/AttributeInstance.kt index d8ab30296..e8d7ae97f 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/temporal/model/AttributeInstance.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/temporal/model/AttributeInstance.kt @@ -85,7 +85,8 @@ data class AttributeInstance private constructor( enum class TemporalProperty(val propertyName: String) { OBSERVED_AT("observedAt"), CREATED_AT("createdAt"), - MODIFIED_AT("modifiedAt"); + MODIFIED_AT("modifiedAt"), + DELETED_AT("deletedAt"); companion object { fun forPropertyName(propertyName: String): TemporalProperty? = diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/temporal/service/AttributeInstanceService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/temporal/service/AttributeInstanceService.kt index 9f1fa6f32..d718bd9ee 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/temporal/service/AttributeInstanceService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/temporal/service/AttributeInstanceService.kt @@ -14,11 +14,13 @@ import com.egm.stellio.search.common.util.toJsonString import com.egm.stellio.search.common.util.toUuid import com.egm.stellio.search.common.util.toZonedDateTime import com.egm.stellio.search.entity.model.Attribute +import com.egm.stellio.search.entity.model.Attribute.AttributeValueType import com.egm.stellio.search.entity.model.AttributeMetadata import com.egm.stellio.search.entity.util.toAttributeMetadata import com.egm.stellio.search.temporal.model.AggregatedAttributeInstanceResult import com.egm.stellio.search.temporal.model.AggregatedAttributeInstanceResult.AggregateResult import com.egm.stellio.search.temporal.model.AttributeInstance +import com.egm.stellio.search.temporal.model.AttributeInstance.TemporalProperty.DELETED_AT import com.egm.stellio.search.temporal.model.AttributeInstance.TemporalProperty.OBSERVED_AT import com.egm.stellio.search.temporal.model.AttributeInstanceResult import com.egm.stellio.search.temporal.model.FullAttributeInstanceResult @@ -36,6 +38,7 @@ import com.egm.stellio.shared.model.ResourceNotFoundException import com.egm.stellio.shared.model.toNgsiLdAttribute import com.egm.stellio.shared.util.INCONSISTENT_VALUES_IN_AGGREGATION_MESSAGE import com.egm.stellio.shared.util.attributeOrInstanceNotFoundMessage +import com.egm.stellio.shared.util.getSubFromSecurityContext import com.egm.stellio.shared.util.ngsiLdDateTime import org.springframework.r2dbc.core.DatabaseClient import org.springframework.r2dbc.core.bind @@ -123,7 +126,7 @@ class AttributeInstanceService( } @Transactional - suspend fun addAttributeInstance( + suspend fun addObservedAttributeInstance( attributeUuid: UUID, attributeMetadata: AttributeMetadata, attributeValues: Map> @@ -137,6 +140,23 @@ class AttributeInstanceService( return create(attributeInstance) } + @Transactional + suspend fun addDeletedAttributeInstance( + attributeUuid: UUID, + value: String, + deletedAt: ZonedDateTime, + attributeValues: Map> + ): Either { + val attributeInstance = AttributeInstance( + attributeUuid = attributeUuid, + timeAndProperty = deletedAt to DELETED_AT, + value = Triple(value, null, null), + payload = attributeValues, + sub = getSubFromSecurityContext().getOrNull() + ) + return create(attributeInstance) + } + suspend fun search( temporalEntitiesQuery: TemporalEntitiesQuery, attribute: Attribute, @@ -248,9 +268,11 @@ class AttributeInstanceService( } else "SELECT temporal_entity_attribute, min(time) as start, max(time) as end, $allAggregates " } else { - val valueColumn = when (attributes[0].attributeValueType) { - Attribute.AttributeValueType.NUMBER -> "measured_value as value" - Attribute.AttributeValueType.GEOMETRY -> "public.ST_AsText(geo_value) as value" + val valueColumn = when { + // for deletedAt, the NGSI-LD Null representation is always stored as string in value column + temporalQuery.timeproperty == DELETED_AT -> "value" + attributes[0].attributeValueType == AttributeValueType.NUMBER -> "measured_value as value" + attributes[0].attributeValueType == AttributeValueType.GEOMETRY -> "public.ST_AsText(geo_value) as value" else -> "value" } val subColumn = diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/temporal/service/TemporalQueryService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/temporal/service/TemporalQueryService.kt index 2371180c1..8f0e742ef 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/temporal/service/TemporalQueryService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/temporal/service/TemporalQueryService.kt @@ -43,12 +43,12 @@ class TemporalQueryService( temporalEntitiesQuery: TemporalEntitiesQuery, sub: Sub? = null ): Either> = either { - entityQueryService.checkEntityExistence(entityId).bind() + val entity = entityQueryService.retrieve(entityId).bind() authorizationService.userCanReadEntity(entityId, sub.toOption()).bind() val attrs = temporalEntitiesQuery.entitiesQuery.attrs val datasetIds = temporalEntitiesQuery.entitiesQuery.datasetId - val attributes = entityAttributeService.getForEntity(entityId, attrs, datasetIds).let { + val attributes = entityAttributeService.getForEntity(entityId, attrs, datasetIds, false).let { if (it.isEmpty()) ResourceNotFoundException( entityOrAttrsNotFoundMessage(entityId.toString(), temporalEntitiesQuery.entitiesQuery.attrs) @@ -56,7 +56,6 @@ class TemporalQueryService( else it.right() }.bind() - val entityPayload = entityQueryService.retrieve(entityId).bind() val origin = calculateOldestTimestamp(entityId, temporalEntitiesQuery, attributes) val scopeHistory = @@ -76,7 +75,7 @@ class TemporalQueryService( fillWithAttributesWithEmptyInstances(attributes, paginatedAttributesWithInstances) TemporalEntityBuilder.buildTemporalEntity( - EntityTemporalResult(entityPayload, scopeHistory, attributesWithInstances), + EntityTemporalResult(entity, scopeHistory, attributesWithInstances), temporalEntitiesQuery ) to range } @@ -94,10 +93,10 @@ class TemporalQueryService( // - timeAt if it is provided // - the oldest value if not (timeAt is optional if querying a temporal entity by id) - if (!temporalEntitiesQuery.withAggregatedValues) - return null + return if (!temporalEntitiesQuery.withAggregatedValues) + null else if (temporalQuery.timeAt != null) - return temporalQuery.timeAt + temporalQuery.timeAt else { val originForAttributes = attributeInstanceService.selectOldestDate(temporalQuery, attributes) @@ -108,7 +107,7 @@ class TemporalQueryService( scopeService.selectOldestDate(entityId, temporalEntitiesQuery.temporalQuery.timeproperty) else null - return when { + when { originForAttributes == null -> originForScope originForScope == null -> originForAttributes else -> minOf(originForAttributes, originForScope) @@ -122,9 +121,11 @@ class TemporalQueryService( ): Either, Int, Range?>> = either { val accessRightFilter = authorizationService.computeAccessRightFilter(sub.toOption()) val attrs = temporalEntitiesQuery.entitiesQuery.attrs - val entitiesIds = entityQueryService.queryEntities(temporalEntitiesQuery.entitiesQuery, accessRightFilter) - val count = entityQueryService.queryEntitiesCount(temporalEntitiesQuery.entitiesQuery, accessRightFilter) - .getOrElse { 0 } + val entitiesIds = + entityQueryService.queryEntities(temporalEntitiesQuery.entitiesQuery, false, accessRightFilter) + val count = + entityQueryService.queryEntitiesCount(temporalEntitiesQuery.entitiesQuery, false, accessRightFilter) + .getOrElse { 0 } // we can have an empty list of entities with a non-zero count (e.g., offset too high) if (entitiesIds.isEmpty()) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/temporal/service/TemporalService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/temporal/service/TemporalService.kt index 25f0939f2..047cfd024 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/temporal/service/TemporalService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/temporal/service/TemporalService.kt @@ -1,6 +1,8 @@ package com.egm.stellio.search.temporal.service import arrow.core.Either +import arrow.core.Either.Left +import arrow.core.Either.Right import arrow.core.raise.either import arrow.core.toOption import com.egm.stellio.search.authorization.service.AuthorizationService @@ -11,12 +13,15 @@ import com.egm.stellio.shared.model.ExpandedAttribute import com.egm.stellio.shared.model.ExpandedAttributes import com.egm.stellio.shared.model.ExpandedEntity import com.egm.stellio.shared.model.ExpandedTerm +import com.egm.stellio.shared.model.NgsiLdEntity import com.egm.stellio.shared.model.addCoreMembers import com.egm.stellio.shared.model.getMemberValueAsDateTime import com.egm.stellio.shared.model.toNgsiLdEntity import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_OBSERVED_AT_PROPERTY import com.egm.stellio.shared.util.Sub +import com.egm.stellio.shared.util.ngsiLdDateTime import org.springframework.stereotype.Service +import org.springframework.transaction.annotation.Transactional import java.net.URI @Service @@ -34,25 +39,28 @@ class TemporalService( jsonLdTemporalEntity: ExpandedEntity, sub: Sub? = null ): Either = either { - val entityDoesNotExist = entityQueryService.checkEntityExistence(entityId, true).isRight() - - if (entityDoesNotExist) { - createTemporalEntity( - entityId, - jsonLdTemporalEntity, - jsonLdTemporalEntity.getAttributes().sorted(), - sub - ).bind() - - CreateOrUpdateResult.CREATED - } else { - upsertTemporalEntity( - entityId, - jsonLdTemporalEntity.getAttributes().sorted(), - sub - ).bind() - - CreateOrUpdateResult.UPSERTED + entityQueryService.isMarkedAsDeleted(entityId).let { + when (it) { + is Left -> { + createTemporalEntity( + entityId, + jsonLdTemporalEntity, + jsonLdTemporalEntity.getAttributes().sorted(), + sub + ).bind() + CreateOrUpdateResult.CREATED + } + is Right -> { + upsertTemporalEntity( + entityId, + jsonLdTemporalEntity, + jsonLdTemporalEntity.getAttributes().sorted(), + it.value, + sub + ).bind() + CreateOrUpdateResult.UPSERTED + } + } } } @@ -64,14 +72,7 @@ class TemporalService( ): Either = either { authorizationService.userCanCreateEntities(sub.toOption()).bind() - // create a view of the entity containing only the most recent instance of each attribute - val expandedEntity = ExpandedEntity( - sortedJsonLdInstances - .keepFirstInstances() - .addCoreMembers(jsonLdTemporalEntity.id, jsonLdTemporalEntity.types) - ) - val ngsiLdEntity = expandedEntity.toNgsiLdEntity().bind() - + val (expandedEntity, ngsiLdEntity) = parseExpandedInstances(sortedJsonLdInstances, jsonLdTemporalEntity).bind() entityService.createEntity(ngsiLdEntity, expandedEntity, sub).bind() entityService.upsertAttributes( entityId, @@ -83,10 +84,17 @@ class TemporalService( internal suspend fun upsertTemporalEntity( entityId: URI, + jsonLdTemporalEntity: ExpandedEntity, sortedJsonLdInstances: ExpandedAttributes, + isDeleted: Boolean, sub: Sub? = null ): Either = either { authorizationService.userCanUpdateEntity(entityId, sub.toOption()).bind() + if (isDeleted) { + val (expandedEntity, ngsiLdEntity) = + parseExpandedInstances(sortedJsonLdInstances, jsonLdTemporalEntity).bind() + entityService.createEntityPayload(ngsiLdEntity, expandedEntity, ngsiLdDateTime()).bind() + } entityService.upsertAttributes( entityId, sortedJsonLdInstances, @@ -94,6 +102,21 @@ class TemporalService( ).bind() } + private suspend fun parseExpandedInstances( + sortedJsonLdInstances: ExpandedAttributes, + jsonLdTemporalEntity: ExpandedEntity + ): Either> = either { + // create a view of the entity containing only the most recent instance of each attribute + val expandedEntity = ExpandedEntity( + sortedJsonLdInstances + .keepFirstInstances() + .addCoreMembers(jsonLdTemporalEntity.id, jsonLdTemporalEntity.types) + ) + val ngsiLdEntity = expandedEntity.toNgsiLdEntity().bind() + + Pair(expandedEntity, ngsiLdEntity) + } + private fun ExpandedAttributes.keepFirstInstances(): ExpandedAttributes = this.mapValues { listOf(it.value.first()) } @@ -124,6 +147,25 @@ class TemporalService( ).bind() } + @Transactional + suspend fun deleteEntity( + entityId: URI, + sub: Sub? = null + ): Either = either { + entityService.permanentlyDeleteEntity(entityId, sub).bind() + } + + @Transactional + suspend fun deleteAttribute( + entityId: URI, + attributeName: ExpandedTerm, + datasetId: URI?, + deleteAll: Boolean = false, + sub: Sub? = null + ): Either = either { + entityService.permanentlyDeleteAttribute(entityId, attributeName, datasetId, deleteAll, sub).bind() + } + suspend fun modifyAttributeInstance( entityId: URI, instanceId: URI, diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/temporal/web/TemporalEntityHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/temporal/web/TemporalEntityHandler.kt index 158ff122f..2b176ef34 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/temporal/web/TemporalEntityHandler.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/temporal/web/TemporalEntityHandler.kt @@ -4,7 +4,6 @@ import arrow.core.Either import arrow.core.left import arrow.core.raise.either import arrow.core.right -import com.egm.stellio.search.entity.service.EntityService import com.egm.stellio.search.temporal.service.TemporalQueryService import com.egm.stellio.search.temporal.service.TemporalService import com.egm.stellio.search.temporal.util.composeTemporalEntitiesQueryFromGet @@ -64,7 +63,6 @@ import java.net.URI class TemporalEntityHandler( private val temporalService: TemporalService, private val temporalQueryService: TemporalQueryService, - private val entityService: EntityService, private val applicationProperties: ApplicationProperties ) : BaseHandler() { @@ -277,7 +275,7 @@ class TemporalEntityHandler( ): ResponseEntity<*> = either { val sub = getSubFromSecurityContext() - entityService.deleteEntity(entityId, sub.getOrNull()).bind() + temporalService.deleteEntity(entityId, sub.getOrNull()).bind() ResponseEntity.status(HttpStatus.NO_CONTENT).build() }.fold( @@ -307,7 +305,7 @@ class TemporalEntityHandler( attrId.checkNameIsNgsiLdSupported().bind() val expandedAttrId = expandJsonLdTerm(attrId, contexts) - entityService.deleteAttribute( + temporalService.deleteAttribute( entityId, expandedAttrId, datasetId, diff --git a/search-service/src/main/resources/db/migration/V0_45__add_deleted_at_temporal_property.sql b/search-service/src/main/resources/db/migration/V0_45__add_deleted_at_temporal_property.sql new file mode 100644 index 000000000..92f72d914 --- /dev/null +++ b/search-service/src/main/resources/db/migration/V0_45__add_deleted_at_temporal_property.sql @@ -0,0 +1,13 @@ +ALTER TABLE temporal_entity_attribute + ADD COLUMN deleted_at timestamp with time zone; + +ALTER TABLE entity_payload + ADD COLUMN deleted_at timestamp with time zone; + +DROP INDEX IF EXISTS temporal_entity_attribute_null_datasetid_uniqueness; + +ALTER TABLE temporal_entity_attribute + DROP CONSTRAINT temporal_entity_attribute_uniqueness; + +ALTER TABLE temporal_entity_attribute + ADD CONSTRAINT temporal_entity_attribute_uniqueness UNIQUE NULLS NOT DISTINCT (entity_id, attribute_name, dataset_id); diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/authorization/listener/IAMListenerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/authorization/listener/IAMListenerTests.kt index 0132af224..cd52b2eb6 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/authorization/listener/IAMListenerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/authorization/listener/IAMListenerTests.kt @@ -348,7 +348,9 @@ class IAMListenerTests { coEvery { entityAccessRightsService.getEntitiesIdsOwnedBySubject("6ad19fe0-fc11-4024-85f2-931c6fa6f7e0") } returns listOf(entityId).right() - coEvery { entityService.deleteEntity(entityId, "6ad19fe0-fc11-4024-85f2-931c6fa6f7e0") } returns Unit.right() + coEvery { + entityService.permanentlyDeleteEntity(entityId, "6ad19fe0-fc11-4024-85f2-931c6fa6f7e0") + } returns Unit.right() iamListener.dispatchIamMessage(subjectDeleteEvent) @@ -360,7 +362,7 @@ class IAMListenerTests { ) } coVerify { - entityService.deleteEntity( + entityService.permanentlyDeleteEntity( eq(entityId), eq("6ad19fe0-fc11-4024-85f2-931c6fa6f7e0") ) diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/AuthorizationServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/AuthorizationServiceTests.kt index 0fd348842..12c3ce2b8 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/AuthorizationServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/AuthorizationServiceTests.kt @@ -40,6 +40,7 @@ class AuthorizationServiceTests { paginationQuery = PaginationQuery(limit = 0, offset = 0), contexts = listOf(applicationProperties.contexts.core) ), + false, listOf(applicationProperties.contexts.core), None ).shouldSucceedWith { diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/EnabledAuthorizationServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/EnabledAuthorizationServiceTests.kt index 2d0cc8d69..edab33f63 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/EnabledAuthorizationServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/EnabledAuthorizationServiceTests.kt @@ -347,7 +347,7 @@ class EnabledAuthorizationServiceTests { @Test fun `it should returned serialized access control entities with a count`() = runTest { coEvery { - entityAccessRightsService.getSubjectAccessRights(any(), any(), any(), any(), any()) + entityAccessRightsService.getSubjectAccessRights(any(), any(), any(), any()) } returns listOf( EntityAccessRights( id = entityId01, @@ -356,7 +356,7 @@ class EnabledAuthorizationServiceTests { ) ).right() coEvery { - entityAccessRightsService.getSubjectAccessRightsCount(any(), any(), any(), any()) + entityAccessRightsService.getSubjectAccessRightsCount(any(), any(), any(), any(), any()) } returns Either.Right(1) coEvery { entityAccessRightsService.getAccessRightsForEntities(any(), any()) @@ -369,6 +369,7 @@ class EnabledAuthorizationServiceTests { contexts = APIC_COMPOUND_CONTEXTS ), contexts = APIC_COMPOUND_CONTEXTS, + includeDeleted = false, sub = Some(subjectUuid) ).shouldSucceedWith { assertEquals(1, it.first) @@ -389,7 +390,7 @@ class EnabledAuthorizationServiceTests { @Test fun `it should returned serialized access control entities with other rigths if user is admin`() = runTest { coEvery { - entityAccessRightsService.getSubjectAccessRights(any(), any(), any(), any(), any()) + entityAccessRightsService.getSubjectAccessRights(any(), any(), any(), any()) } returns listOf( EntityAccessRights( id = entityId01, @@ -403,7 +404,7 @@ class EnabledAuthorizationServiceTests { ) ).right() coEvery { - entityAccessRightsService.getSubjectAccessRightsCount(any(), any(), any(), any()) + entityAccessRightsService.getSubjectAccessRightsCount(any(), any(), any(), any(), any()) } returns Either.Right(1) coEvery { entityAccessRightsService.getAccessRightsForEntities(any(), any()) @@ -424,6 +425,7 @@ class EnabledAuthorizationServiceTests { paginationQuery = PaginationQuery(limit = 10, offset = 0), contexts = APIC_COMPOUND_CONTEXTS ), + includeDeleted = false, contexts = APIC_COMPOUND_CONTEXTS, sub = Some(subjectUuid) ).shouldSucceedWith { @@ -446,10 +448,10 @@ class EnabledAuthorizationServiceTests { @Test fun `it should return serialized access control entities with other rigths if user is owner`() = runTest { coEvery { - entityAccessRightsService.getSubjectAccessRights(any(), any(), any(), any(), any()) + entityAccessRightsService.getSubjectAccessRights(any(), any(), any(), any()) } returns listOf(EntityAccessRights(id = entityId01, types = listOf(BEEHIVE_TYPE), right = IS_OWNER)).right() coEvery { - entityAccessRightsService.getSubjectAccessRightsCount(any(), any(), any(), any()) + entityAccessRightsService.getSubjectAccessRightsCount(any(), any(), any(), any(), any()) } returns Either.Right(1) coEvery { entityAccessRightsService.getAccessRightsForEntities(any(), any()) @@ -476,6 +478,7 @@ class EnabledAuthorizationServiceTests { paginationQuery = PaginationQuery(limit = 10, offset = 0), contexts = APIC_COMPOUND_CONTEXTS ), + includeDeleted = false, contexts = AUTHZ_TEST_COMPOUND_CONTEXTS, sub = Some(subjectUuid) ).shouldSucceedWith { diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/EntityAccessRightsServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/EntityAccessRightsServiceTests.kt index 6365035a4..15a71eecb 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/EntityAccessRightsServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/EntityAccessRightsServiceTests.kt @@ -7,8 +7,10 @@ import com.egm.stellio.search.authorization.getSubjectInfoForGroup import com.egm.stellio.search.authorization.getSubjectInfoForUser import com.egm.stellio.search.authorization.model.SubjectAccessRight import com.egm.stellio.search.authorization.model.SubjectReferential +import com.egm.stellio.search.entity.model.EntitiesQueryFromGet import com.egm.stellio.search.entity.model.Entity import com.egm.stellio.search.entity.service.EntityService +import com.egm.stellio.search.support.WithKafkaContainer import com.egm.stellio.search.support.WithTimescaleContainer import com.egm.stellio.search.support.buildSapAttribute import com.egm.stellio.shared.model.AccessDeniedException @@ -58,7 +60,7 @@ import java.util.UUID @SpringBootTest @ActiveProfiles("test") -class EntityAccessRightsServiceTests : WithTimescaleContainer { +class EntityAccessRightsServiceTests : WithTimescaleContainer, WithKafkaContainer() { @Autowired private lateinit var entityAccessRightsService: EntityAccessRightsService @@ -245,7 +247,10 @@ class EntityAccessRightsServiceTests : WithTimescaleContainer { entityAccessRightsService.getSubjectAccessRights( Some(userUuid), emptyList(), - paginationQuery = PaginationQuery(limit = 100, offset = 0) + entitiesQuery = EntitiesQueryFromGet( + paginationQuery = PaginationQuery(limit = 100, offset = 0), + contexts = emptyList() + ) ).shouldSucceedWith { assertEquals(1, it.size) val entityAccessControl = it[0] @@ -274,7 +279,10 @@ class EntityAccessRightsServiceTests : WithTimescaleContainer { entityAccessRightsService.getSubjectAccessRights( Some(userUuid), emptyList(), - paginationQuery = PaginationQuery(limit = 100, offset = 0) + entitiesQuery = EntitiesQueryFromGet( + paginationQuery = PaginationQuery(limit = 100, offset = 0), + contexts = emptyList() + ) ).shouldSucceedWith { assertEquals(1, it.size) val entityAccessControl = it[0] @@ -305,7 +313,10 @@ class EntityAccessRightsServiceTests : WithTimescaleContainer { entityAccessRightsService.getSubjectAccessRights( Some(userUuid), emptyList(), - paginationQuery = PaginationQuery(limit = 100, offset = 0) + entitiesQuery = EntitiesQueryFromGet( + paginationQuery = PaginationQuery(limit = 100, offset = 0), + contexts = emptyList() + ) ).shouldSucceedWith { assertEquals(2, it.size) it.forEach { entityAccessControl -> @@ -336,8 +347,11 @@ class EntityAccessRightsServiceTests : WithTimescaleContainer { entityAccessRightsService.getSubjectAccessRights( Some(userUuid), emptyList(), - BEEHIVE_TYPE, - paginationQuery = PaginationQuery(limit = 100, offset = 0) + entitiesQuery = EntitiesQueryFromGet( + typeSelection = BEEHIVE_TYPE, + paginationQuery = PaginationQuery(limit = 100, offset = 0), + contexts = emptyList() + ) ).shouldSucceedWith { assertEquals(1, it.size) val entityAccessControl = it[0] @@ -369,9 +383,11 @@ class EntityAccessRightsServiceTests : WithTimescaleContainer { entityAccessRightsService.getSubjectAccessRights( Some(userUuid), emptyList(), - null, - setOf(entityId01, entityId02), - paginationQuery = PaginationQuery(limit = 100, offset = 0) + entitiesQuery = EntitiesQueryFromGet( + ids = setOf(entityId01, entityId02), + paginationQuery = PaginationQuery(limit = 100, offset = 0), + contexts = emptyList() + ) ).shouldSucceedWith { assertEquals(1, it.size) assertEquals(entityId01, it[0].id) @@ -402,9 +418,12 @@ class EntityAccessRightsServiceTests : WithTimescaleContainer { entityAccessRightsService.getSubjectAccessRights( Some(userUuid), emptyList(), - BEEHIVE_TYPE, - setOf(entityId01, entityId03), - paginationQuery = PaginationQuery(limit = 100, offset = 0) + entitiesQuery = EntitiesQueryFromGet( + ids = setOf(entityId01, entityId02), + typeSelection = BEEHIVE_TYPE, + paginationQuery = PaginationQuery(limit = 100, offset = 0), + contexts = emptyList() + ) ).shouldSucceedWith { assertEquals(1, it.size) assertEquals(entityId01, it[0].id) @@ -435,7 +454,10 @@ class EntityAccessRightsServiceTests : WithTimescaleContainer { entityAccessRightsService.getSubjectAccessRights( Some(userUuid), listOf(AccessRight.CAN_WRITE), - paginationQuery = PaginationQuery(limit = 100, offset = 0) + entitiesQuery = EntitiesQueryFromGet( + paginationQuery = PaginationQuery(limit = 100, offset = 0), + contexts = emptyList() + ) ).shouldSucceedWith { assertEquals(1, it.size) val entityAccessControl = it[0] @@ -464,8 +486,11 @@ class EntityAccessRightsServiceTests : WithTimescaleContainer { entityAccessRightsService.getSubjectAccessRights( Some(userUuid), listOf(AccessRight.CAN_WRITE), - "$BEEHIVE_TYPE,$APIARY_TYPE", - paginationQuery = PaginationQuery(limit = 100, offset = 0) + entitiesQuery = EntitiesQueryFromGet( + typeSelection = "$BEEHIVE_TYPE,$APIARY_TYPE", + paginationQuery = PaginationQuery(limit = 100, offset = 0), + contexts = emptyList() + ) ).shouldSucceedWith { assertEquals(1, it.size) val entityAccessControl = it[0] @@ -493,8 +518,11 @@ class EntityAccessRightsServiceTests : WithTimescaleContainer { entityAccessRightsService.getSubjectAccessRights( Some(userUuid), emptyList(), - BEEHIVE_TYPE, - paginationQuery = PaginationQuery(limit = 100, offset = 0) + entitiesQuery = EntitiesQueryFromGet( + typeSelection = BEEHIVE_TYPE, + paginationQuery = PaginationQuery(limit = 100, offset = 0), + contexts = emptyList() + ) ).shouldSucceedWith { assertEquals(1, it.size) val entityAccessControl = it[0] @@ -503,6 +531,66 @@ class EntityAccessRightsServiceTests : WithTimescaleContainer { } } + @Test + fun `getSubjectAccessRights should include deleted entities if it is asked for`() = runTest { + createEntityPayload(entityId01, setOf(BEEHIVE_TYPE), AUTH_READ) + entityAccessRightsService.setRoleOnEntity(userUuid, entityId01, AccessRight.CAN_WRITE).shouldSucceed() + + createEntityPayload(entityId02, setOf(BEEHIVE_TYPE)) + entityAccessRightsService.setRoleOnEntity(userUuid, entityId02, AccessRight.CAN_ADMIN).shouldSucceed() + entityService.deleteEntity(entityId02, userUuid).shouldSucceed() + + entityAccessRightsService.getSubjectAccessRights( + Some(userUuid), + emptyList(), + entitiesQuery = EntitiesQueryFromGet( + paginationQuery = PaginationQuery(limit = 100, offset = 0), + contexts = emptyList() + ), + includeDeleted = true + ).shouldSucceedWith { entityAccessRights -> + assertEquals(2, entityAccessRights.size) + assertEquals(1, entityAccessRights.filter { it.isDeleted }.size) + } + + entityAccessRightsService.getSubjectAccessRightsCount( + Some(userUuid), + emptyList(), + includeDeleted = true + ).shouldSucceedWith { + assertEquals(2, it) + } + } + + @Test + fun `getSubjectAccessRights should not include deleted entities if it is not asked for`() = runTest { + createEntityPayload(entityId01, setOf(BEEHIVE_TYPE)) + entityAccessRightsService.setRoleOnEntity(userUuid, entityId01, AccessRight.CAN_WRITE).shouldSucceed() + + createEntityPayload(entityId02, setOf(BEEHIVE_TYPE)) + entityAccessRightsService.setRoleOnEntity(userUuid, entityId02, AccessRight.CAN_ADMIN).shouldSucceed() + entityService.deleteEntity(entityId02, userUuid).shouldSucceed() + + entityAccessRightsService.getSubjectAccessRights( + Some(userUuid), + emptyList(), + entitiesQuery = EntitiesQueryFromGet( + paginationQuery = PaginationQuery(limit = 100, offset = 0), + contexts = emptyList() + ) + ).shouldSucceedWith { entityAccessRights -> + assertEquals(1, entityAccessRights.size) + assertEquals(0, entityAccessRights.filter { it.isDeleted }.size) + } + + entityAccessRightsService.getSubjectAccessRightsCount( + Some(userUuid), + emptyList() + ).shouldSucceedWith { + assertEquals(1, it) + } + } + @Test fun `it should return nothing when list of entities is empty`() = runTest { entityAccessRightsService.getAccessRightsForEntities(Some(userUuid), emptyList()) diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/SubjectReferentialServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/SubjectReferentialServiceTests.kt index 8d09e591a..5ec77a83b 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/SubjectReferentialServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/authorization/service/SubjectReferentialServiceTests.kt @@ -6,6 +6,7 @@ import com.egm.stellio.search.authorization.getSubjectInfoForGroup import com.egm.stellio.search.authorization.getSubjectInfoForUser import com.egm.stellio.search.authorization.model.SubjectReferential import com.egm.stellio.search.support.EMPTY_JSON_PAYLOAD +import com.egm.stellio.search.support.WithKafkaContainer import com.egm.stellio.search.support.WithTimescaleContainer import com.egm.stellio.shared.model.AccessDeniedException import com.egm.stellio.shared.util.ADMIN_ROLES @@ -36,7 +37,7 @@ import java.util.UUID @SpringBootTest @ActiveProfiles("test") -class SubjectReferentialServiceTests : WithTimescaleContainer { +class SubjectReferentialServiceTests : WithTimescaleContainer, WithKafkaContainer() { @Autowired private lateinit var subjectReferentialService: SubjectReferentialService diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/authorization/web/EntityAccessControlHandlerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/authorization/web/EntityAccessControlHandlerTests.kt index 645272abd..fbd990a86 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/authorization/web/EntityAccessControlHandlerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/authorization/web/EntityAccessControlHandlerTests.kt @@ -596,7 +596,7 @@ class EntityAccessControlHandlerTests { @Test fun `get authorized entities should return 200 and the number of results if requested limit is 0`() { coEvery { - authorizationService.getAuthorizedEntities(any(), any(), any()) + authorizationService.getAuthorizedEntities(any(), any(), any(), any()) } returns Pair(3, emptyList()).right() webClient.get() @@ -610,7 +610,7 @@ class EntityAccessControlHandlerTests { @Test fun `get authorized entities should return 200 and empty response if requested offset does not exist`() { coEvery { - authorizationService.getAuthorizedEntities(any(), any(), any()) + authorizationService.getAuthorizedEntities(any(), any(), any(), any()) } returns Pair(0, emptyList()).right() webClient.get() @@ -620,10 +620,44 @@ class EntityAccessControlHandlerTests { .expectBody().json("[]") } + @Test + fun `get authorized entities should not ask for deleted entities if includeDeleted query param is not provided`() { + coEvery { + authorizationService.getAuthorizedEntities(any(), any(), any(), any()) + } returns Pair(0, emptyList()).right() + + webClient.get() + .uri("/ngsi-ld/v1/entityAccessControl/entities") + .exchange() + .expectStatus().isOk + .expectBody().json("[]") + + coVerify { + authorizationService.getAuthorizedEntities(any(), false, any(), any()) + } + } + + @Test + fun `get authorized entities should ask for deleted entities if includeDeleted query param is true`() { + coEvery { + authorizationService.getAuthorizedEntities(any(), any(), any(), any()) + } returns Pair(0, emptyList()).right() + + webClient.get() + .uri("/ngsi-ld/v1/entityAccessControl/entities?includeDeleted=true") + .exchange() + .expectStatus().isOk + .expectBody().json("[]") + + coVerify { + authorizationService.getAuthorizedEntities(any(), true, any(), any()) + } + } + @Test fun `get authorized entities should return entities I have a right on`() = runTest { coEvery { - authorizationService.getAuthorizedEntities(any(), any(), any()) + authorizationService.getAuthorizedEntities(any(), any(), any(), any()) } returns Pair( 2, listOf( @@ -697,7 +731,7 @@ class EntityAccessControlHandlerTests { @Test fun `get authorized entities should return 204 if authentication is not enabled`() { coEvery { - authorizationService.getAuthorizedEntities(any(), any(), any()) + authorizationService.getAuthorizedEntities(any(), any(), any(), any()) } returns Pair(-1, emptyList()).right() webClient.get() diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationServiceTests.kt index 865e11889..9f99f0a4d 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationServiceTests.kt @@ -5,6 +5,7 @@ import com.egm.stellio.search.csr.model.CSRFilters import com.egm.stellio.search.csr.model.ContextSourceRegistration import com.egm.stellio.search.csr.model.ContextSourceRegistration.Companion.notFoundMessage import com.egm.stellio.search.csr.model.Operation +import com.egm.stellio.search.support.WithKafkaContainer import com.egm.stellio.search.support.WithTimescaleContainer import com.egm.stellio.shared.model.AlreadyExistsException import com.egm.stellio.shared.model.ResourceNotFoundException @@ -33,7 +34,7 @@ import java.util.UUID @SpringBootTest @ActiveProfiles("test") @TestPropertySource(properties = ["application.authentication.enabled=false"]) -class ContextSourceRegistrationServiceTests : WithTimescaleContainer { +class ContextSourceRegistrationServiceTests : WithTimescaleContainer, WithKafkaContainer() { @Autowired private lateinit var contextSourceRegistrationService: ContextSourceRegistrationService diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/discovery/service/AttributeServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/discovery/service/AttributeServiceTests.kt index cd43ffb21..5f9508c19 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/discovery/service/AttributeServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/discovery/service/AttributeServiceTests.kt @@ -52,7 +52,7 @@ import java.time.ZoneOffset @SpringBootTest @ActiveProfiles("test") -class AttributeServiceTests : WithTimescaleContainer, WithKafkaContainer { +class AttributeServiceTests : WithTimescaleContainer, WithKafkaContainer() { @Autowired private lateinit var attributeService: AttributeService diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/discovery/service/EntityTypeServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/discovery/service/EntityTypeServiceTests.kt index abae6170b..a052910d4 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/discovery/service/EntityTypeServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/discovery/service/EntityTypeServiceTests.kt @@ -59,7 +59,7 @@ import org.springframework.test.context.ActiveProfiles @SpringBootTest @ActiveProfiles("test") -class EntityTypeServiceTests : WithTimescaleContainer, WithKafkaContainer { +class EntityTypeServiceTests : WithTimescaleContainer, WithKafkaContainer() { @Autowired private lateinit var entityTypeService: EntityTypeService diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/entity/listener/ObservationEventListenerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/entity/listener/ObservationEventListenerTests.kt index a464ec6c9..f9b6a6bc3 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/entity/listener/ObservationEventListenerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/entity/listener/ObservationEventListenerTests.kt @@ -2,26 +2,23 @@ package com.egm.stellio.search.entity.listener import arrow.core.right import com.egm.stellio.search.entity.model.NotUpdatedDetails -import com.egm.stellio.search.entity.model.UpdateOperationResult +import com.egm.stellio.search.entity.model.OperationStatus import com.egm.stellio.search.entity.model.UpdateResult -import com.egm.stellio.search.entity.model.UpdatedDetails import com.egm.stellio.search.entity.service.EntityEventService import com.egm.stellio.search.entity.service.EntityService -import com.egm.stellio.shared.model.ExpandedEntity -import com.egm.stellio.shared.model.NgsiLdEntity import com.egm.stellio.shared.util.BEEHIVE_TYPE import com.egm.stellio.shared.util.TEMPERATURE_PROPERTY import com.egm.stellio.shared.util.loadSampleData import com.egm.stellio.shared.util.toUri import com.ninjasquad.springmockk.MockkBean import io.mockk.called +import io.mockk.clearAllMocks import io.mockk.coEvery import io.mockk.coVerify -import io.mockk.every -import io.mockk.mockkClass import io.mockk.verify import kotlinx.coroutines.Job import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertDoesNotThrow import org.springframework.beans.factory.annotation.Autowired @@ -35,21 +32,26 @@ class ObservationEventListenerTests { @Autowired private lateinit var observationEventListener: ObservationEventListener - @MockkBean(relaxed = true) + @MockkBean private lateinit var entityService: EntityService - @MockkBean(relaxed = true) + @MockkBean private lateinit var entityEventService: EntityEventService private val expectedEntityId = "urn:ngsi-ld:BeeHive:01".toUri() private val expectedTemperatureDatasetId = "urn:ngsi-ld:Dataset:WeatherApi".toUri() + @BeforeEach + fun clearMocks() { + clearAllMocks() + } + @Test fun `it should parse and transmit an ENTITY_CREATE event`() = runTest { val observationEvent = loadSampleData("events/entity/entityCreateEvent.json") coEvery { - entityService.createEntity(any(), any(), any()) + entityService.createEntity(any(), any(), any()) } returns Unit.right() coEvery { entityEventService.publishEntityCreateEvent(any(), any(), any()) } returns Job() @@ -57,7 +59,7 @@ class ObservationEventListenerTests { coVerify { entityService.createEntity( - any(), + any(), any(), eq("0123456789-1234-5678-987654321") ) @@ -79,23 +81,17 @@ class ObservationEventListenerTests { coEvery { entityService.partialUpdateAttribute(any(), any(), any()) } returns UpdateResult( - updated = arrayListOf( - UpdatedDetails( - TEMPERATURE_PROPERTY, - expectedTemperatureDatasetId, - UpdateOperationResult.UPDATED - ) - ), - notUpdated = arrayListOf() + updated = listOf(TEMPERATURE_PROPERTY), + notUpdated = emptyList() ).right() coEvery { - entityEventService.publishAttributeChangeEvents(any(), any(), any(), any(), any()) + entityEventService.publishAttributeChangeEvents(any(), any(), any()) } returns Job() observationEventListener.dispatchObservationMessage(observationEvent) - coVerify { + coVerify(timeout = 1000L) { entityService.partialUpdateAttribute( expectedEntityId, match { it.first == TEMPERATURE_PROPERTY }, @@ -106,14 +102,12 @@ class ObservationEventListenerTests { entityEventService.publishAttributeChangeEvents( null, eq(expectedEntityId), - match { it.containsKey(TEMPERATURE_PROPERTY) }, match { - it.updated.size == 1 && - it.updated[0].attributeName == TEMPERATURE_PROPERTY && - it.updated[0].datasetId == expectedTemperatureDatasetId && - it.updated[0].updateOperationResult == UpdateOperationResult.UPDATED - }, - eq(false) + it.size == 1 && + it[0].attributeName == TEMPERATURE_PROPERTY && + it[0].datasetId == expectedTemperatureDatasetId && + it[0].operationStatus == OperationStatus.UPDATED + } ) } } @@ -131,6 +125,9 @@ class ObservationEventListenerTests { observationEventListener.dispatchObservationMessage(observationEvent) + coVerify { + entityService.partialUpdateAttribute(any(), any(), any()) + } verify { entityEventService wasNot called } } @@ -141,19 +138,11 @@ class ObservationEventListenerTests { coEvery { entityService.appendAttributes(any(), any(), any(), any()) } returns UpdateResult( - listOf( - UpdatedDetails( - TEMPERATURE_PROPERTY, - expectedTemperatureDatasetId, - UpdateOperationResult.APPENDED - ) - ), + listOf(TEMPERATURE_PROPERTY), emptyList() ).right() - val mockedExpandedEntity = mockkClass(ExpandedEntity::class, relaxed = true) - every { mockedExpandedEntity.types } returns listOf(BEEHIVE_TYPE) coEvery { - entityEventService.publishAttributeChangeEvents(any(), any(), any(), any(), any()) + entityEventService.publishAttributeChangeEvents(any(), any(), any()) } returns Job() observationEventListener.dispatchObservationMessage(observationEvent) @@ -171,15 +160,11 @@ class ObservationEventListenerTests { null, eq(expectedEntityId), match { - it.containsKey(TEMPERATURE_PROPERTY) - }, - match { - it.updated.size == 1 && - it.updated[0].updateOperationResult == UpdateOperationResult.APPENDED && - it.updated[0].attributeName == TEMPERATURE_PROPERTY && - it.updated[0].datasetId == expectedTemperatureDatasetId - }, - eq(true) + it.size == 1 && + it[0].operationStatus == OperationStatus.APPENDED && + it[0].attributeName == TEMPERATURE_PROPERTY && + it[0].datasetId == expectedTemperatureDatasetId + } ) } } diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/entity/model/UpdateResultTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/entity/model/UpdateResultTests.kt index d46951752..da0933c18 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/entity/model/UpdateResultTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/entity/model/UpdateResultTests.kt @@ -1,6 +1,5 @@ package com.egm.stellio.search.entity.model -import com.egm.stellio.shared.util.toUri import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test @@ -9,15 +8,15 @@ class UpdateResultTests { @Test fun `it should find the successful update operation results`() { - assertTrue(UpdateOperationResult.UPDATED.isSuccessResult()) - assertTrue(UpdateOperationResult.APPENDED.isSuccessResult()) - assertTrue(UpdateOperationResult.REPLACED.isSuccessResult()) + assertTrue(OperationStatus.UPDATED.isSuccessResult()) + assertTrue(OperationStatus.APPENDED.isSuccessResult()) + assertTrue(OperationStatus.REPLACED.isSuccessResult()) + assertTrue(OperationStatus.IGNORED.isSuccessResult()) } @Test fun `it should find the failed update operation results`() { - assertFalse(UpdateOperationResult.FAILED.isSuccessResult()) - assertFalse(UpdateOperationResult.IGNORED.isSuccessResult()) + assertFalse(OperationStatus.FAILED.isSuccessResult()) } @Test @@ -25,9 +24,7 @@ class UpdateResultTests { val updateResult = UpdateResult( notUpdated = emptyList(), - updated = listOf( - UpdatedDetails("attributeName", "urn:ngsi-ld:Entity:01".toUri(), UpdateOperationResult.UPDATED) - ) + updated = listOf("attributeName") ) assertTrue(updateResult.isSuccessful()) @@ -37,12 +34,8 @@ class UpdateResultTests { fun `it should find a failed update result if there is one not updated attribute`() { val updateResult = UpdateResult( - notUpdated = listOf( - NotUpdatedDetails("attributeName", "attribute is malformed") - ), - updated = listOf( - UpdatedDetails("attributeName", "urn:ngsi-ld:Entity:01".toUri(), UpdateOperationResult.UPDATED) - ) + notUpdated = listOf(NotUpdatedDetails("attributeName", "attribute is malformed")), + updated = listOf("attributeName") ) assertFalse(updateResult.isSuccessful()) @@ -52,11 +45,10 @@ class UpdateResultTests { fun `it should find a failed update result if an attribute update has failed`() { val updateResult = UpdateResult( - notUpdated = emptyList(), - updated = listOf( - UpdatedDetails("attributeName", "urn:ngsi-ld:Entity:01".toUri(), UpdateOperationResult.UPDATED), - UpdatedDetails("attributeName", "urn:ngsi-ld:Entity:01".toUri(), UpdateOperationResult.FAILED) - ) + notUpdated = listOf( + NotUpdatedDetails("failedAttributeName", "attribute does not exist") + ), + updated = listOf("succeededAttributeName") ) assertFalse(updateResult.isSuccessful()) diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityAttributeServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityAttributeServiceTests.kt index f0ede34f6..75ca618e1 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityAttributeServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityAttributeServiceTests.kt @@ -4,7 +4,9 @@ import arrow.core.right import com.egm.stellio.search.entity.model.Attribute import com.egm.stellio.search.entity.model.AttributeMetadata import com.egm.stellio.search.entity.model.Entity -import com.egm.stellio.search.entity.model.UpdateOperationResult +import com.egm.stellio.search.entity.model.FailedAttributeOperationResult +import com.egm.stellio.search.entity.model.OperationStatus +import com.egm.stellio.search.entity.model.getSucceededOperations import com.egm.stellio.search.support.EMPTY_JSON_PAYLOAD import com.egm.stellio.search.support.WithKafkaContainer import com.egm.stellio.search.support.WithTimescaleContainer @@ -18,8 +20,10 @@ import com.egm.stellio.shared.util.BEEHIVE_TYPE import com.egm.stellio.shared.util.INCOMING_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DEFAULT_VOCAB +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_NULL import com.egm.stellio.shared.util.JsonLdUtils.expandAttribute import com.egm.stellio.shared.util.JsonUtils.serializeObject +import com.egm.stellio.shared.util.NGSILD_TEST_CORE_CONTEXTS import com.egm.stellio.shared.util.OUTGOING_PROPERTY import com.egm.stellio.shared.util.TEMPERATURE_PROPERTY import com.egm.stellio.shared.util.assertJsonPayloadsAreEqual @@ -56,7 +60,7 @@ import java.time.ZonedDateTime @SpringBootTest @ActiveProfiles("test") -class EntityAttributeServiceTests : WithTimescaleContainer, WithKafkaContainer { +class EntityAttributeServiceTests : WithTimescaleContainer, WithKafkaContainer() { @Autowired @SpykBean @@ -260,6 +264,9 @@ class EntityAttributeServiceTests : WithTimescaleContainer, WithKafkaContainer { val rawEntity = loadSampleData() coEvery { attributeInstanceService.create(any()) } returns Unit.right() + coEvery { + attributeInstanceService.addDeletedAttributeInstance(any(), any(), any(), any()) + } returns Unit.right() entityAttributeService.createAttributes(rawEntity, APIC_COMPOUND_CONTEXTS) .shouldSucceed() @@ -387,12 +394,12 @@ class EntityAttributeServiceTests : WithTimescaleContainer, WithKafkaContainer { createdAt, null, null - ).shouldSucceedWith { updateResult -> - val updatedDetails = updateResult.updated - assertEquals(6, updatedDetails.size) - assertEquals(4, updatedDetails.filter { it.updateOperationResult == UpdateOperationResult.UPDATED }.size) - assertEquals(2, updatedDetails.filter { it.updateOperationResult == UpdateOperationResult.APPENDED }.size) - val newAttributes = updatedDetails.filter { it.updateOperationResult == UpdateOperationResult.APPENDED } + ).shouldSucceedWith { operationResults -> + val successfulOperations = operationResults.getSucceededOperations() + assertEquals(6, successfulOperations.size) + assertEquals(4, successfulOperations.filter { it.operationStatus == OperationStatus.UPDATED }.size) + assertEquals(2, successfulOperations.filter { it.operationStatus == OperationStatus.APPENDED }.size) + val newAttributes = successfulOperations.filter { it.operationStatus == OperationStatus.APPENDED } .map { it.attributeName } assertTrue(newAttributes.containsAll(listOf(OUTGOING_PROPERTY, TEMPERATURE_PROPERTY))) } @@ -454,10 +461,10 @@ class EntityAttributeServiceTests : WithTimescaleContainer, WithKafkaContainer { createdAt, observedAt, null - ).shouldSucceedWith { updateResult -> - val updatedDetails = updateResult.updated - assertEquals(1, updatedDetails.size) - assertEquals(1, updatedDetails.filter { it.updateOperationResult == UpdateOperationResult.UPDATED }.size) + ).shouldSucceedWith { operationResults -> + val successfulOperations = operationResults.getSucceededOperations() + assertEquals(1, successfulOperations.size) + assertEquals(1, successfulOperations.filter { it.operationStatus == OperationStatus.UPDATED }.size) } coVerify(exactly = 1) { @@ -472,11 +479,149 @@ class EntityAttributeServiceTests : WithTimescaleContainer, WithKafkaContainer { } } + @Test + fun `it should delete an attribute in merge operation if its value is NGSI-LD null`() = runTest { + val rawEntity = loadSampleData() + + coEvery { attributeInstanceService.create(any()) } returns Unit.right() + coEvery { + attributeInstanceService.addDeletedAttributeInstance(any(), any(), any(), any()) + } returns Unit.right() + + entityAttributeService.createAttributes(rawEntity, APIC_COMPOUND_CONTEXTS).shouldSucceed() + + val createdAt = ngsiLdDateTime() + val propertyToDelete = loadSampleData("fragments/beehive_mergeAttribute_null.json") + val expandedAttributes = JsonLdUtils.expandAttributes(propertyToDelete, APIC_COMPOUND_CONTEXTS) + val ngsiLdAttributes = expandedAttributes.toMap().toNgsiLdAttributes().shouldSucceedAndResult() + entityAttributeService.mergeAttributes( + beehiveTestCId, + ngsiLdAttributes, + expandedAttributes, + createdAt, + null, + null + ).shouldSucceedWith { operationResults -> + val successfulOperations = operationResults.getSucceededOperations() + assertEquals(1, successfulOperations.size) + assertEquals(1, successfulOperations.filter { it.operationStatus == OperationStatus.DELETED }.size) + } + + coVerify(exactly = 1) { + attributeInstanceService.addDeletedAttributeInstance( + any(), + NGSILD_NULL, + createdAt, + expandAttribute( + INCOMING_PROPERTY, + """ + { + "type": "Property", + "value": "urn:ngsi-ld:null" + } + """.trimIndent(), + NGSILD_TEST_CORE_CONTEXTS + ).second[0] + ) + } + } + + @Test + fun `it should delete an attribute in update operation if its value is NGSI-LD null`() = runTest { + val rawEntity = loadSampleData() + + coEvery { attributeInstanceService.create(any()) } returns Unit.right() + coEvery { + attributeInstanceService.addDeletedAttributeInstance(any(), any(), any(), any()) + } returns Unit.right() + + entityAttributeService.createAttributes(rawEntity, APIC_COMPOUND_CONTEXTS).shouldSucceed() + + val createdAt = ngsiLdDateTime() + val propertyToDelete = loadSampleData("fragments/beehive_mergeAttribute_null.json") + val expandedAttributes = JsonLdUtils.expandAttributes(propertyToDelete, APIC_COMPOUND_CONTEXTS) + val ngsiLdAttributes = expandedAttributes.toMap().toNgsiLdAttributes().shouldSucceedAndResult() + entityAttributeService.updateAttributes( + beehiveTestCId, + ngsiLdAttributes, + expandedAttributes, + createdAt, + null + ).shouldSucceedWith { operationResults -> + val successfulOperations = operationResults.getSucceededOperations() + assertEquals(1, successfulOperations.size) + assertEquals(1, successfulOperations.filter { it.operationStatus == OperationStatus.DELETED }.size) + } + + coVerify(exactly = 1) { + attributeInstanceService.addDeletedAttributeInstance( + any(), + NGSILD_NULL, + createdAt, + expandAttribute( + INCOMING_PROPERTY, + """ + { + "type": "Property", + "value": "urn:ngsi-ld:null" + } + """.trimIndent(), + NGSILD_TEST_CORE_CONTEXTS + ).second[0] + ) + } + } + + @Test + fun `it should delete an attribute in partial attribute update operation if its value is NGSI-LD null`() = runTest { + val rawEntity = loadSampleData() + + coEvery { attributeInstanceService.create(any()) } returns Unit.right() + coEvery { + attributeInstanceService.addDeletedAttributeInstance(any(), any(), any(), any()) + } returns Unit.right() + + entityAttributeService.createAttributes(rawEntity, APIC_COMPOUND_CONTEXTS).shouldSucceed() + + val createdAt = ngsiLdDateTime() + val propertyToDelete = loadSampleData("fragments/beehive_mergeAttribute_null_fragment.json") + val expandedAttribute = expandAttribute(INCOMING_PROPERTY, propertyToDelete, APIC_COMPOUND_CONTEXTS) + entityAttributeService.partialUpdateAttribute( + beehiveTestCId, + expandedAttribute, + createdAt, + null + ).shouldSucceedWith { operationResult -> + assertEquals(OperationStatus.DELETED, operationResult.operationStatus) + } + + coVerify(exactly = 1) { + attributeInstanceService.addDeletedAttributeInstance( + any(), + NGSILD_NULL, + createdAt, + expandAttribute( + INCOMING_PROPERTY, + """ + { + "type": "Property", + "value": "urn:ngsi-ld:null" + } + """.trimIndent(), + NGSILD_TEST_CORE_CONTEXTS + ).second[0] + ) + } + } + @Test fun `it should replace an entity attribute`() = runTest { val rawEntity = loadSampleData() coEvery { attributeInstanceService.create(any()) } returns Unit.right() + coEvery { + attributeInstanceService.addDeletedAttributeInstance(any(), any(), any(), any()) + } returns Unit.right() entityAttributeService.createAttributes(rawEntity, APIC_COMPOUND_CONTEXTS) .shouldSucceed() @@ -526,11 +671,9 @@ class EntityAttributeServiceTests : WithTimescaleContainer, WithKafkaContainer { expandedAttribute, replacedAt, null - ).shouldSucceedWith { - assertTrue(it.updated.isEmpty()) - assertEquals(1, it.notUpdated.size) - val notUpdatedDetails = it.notUpdated.first() - assertEquals(NGSILD_DEFAULT_VOCAB + "unknown", notUpdatedDetails.attributeName) + ).shouldSucceedWith { operationResult -> + assertInstanceOf(FailedAttributeOperationResult::class.java, operationResult) + assertEquals(NGSILD_DEFAULT_VOCAB + "unknown", operationResult.attributeName) } } @@ -581,34 +724,57 @@ class EntityAttributeServiceTests : WithTimescaleContainer, WithKafkaContainer { } @Test - fun `it should delete a temporal attribute references`() = runTest { + fun `it should flag and audit a deleted attribute`() = runTest { val rawEntity = loadSampleData("beehive_two_temporal_properties.jsonld") coEvery { attributeInstanceService.create(any()) } returns Unit.right() - coEvery { attributeInstanceService.deleteInstancesOfAttribute(any(), any(), any()) } returns Unit.right() + coEvery { + attributeInstanceService.addDeletedAttributeInstance(any(), any(), any(), any()) + } returns Unit.right() entityAttributeService.createAttributes(rawEntity, APIC_COMPOUND_CONTEXTS) + val deletedAt = ngsiLdDateTime() entityAttributeService.deleteAttribute( beehiveTestDId, INCOMING_PROPERTY, - null + null, + false, + deletedAt ).shouldSucceed() coVerify { - attributeInstanceService.deleteInstancesOfAttribute(eq(beehiveTestDId), eq(INCOMING_PROPERTY), null) + attributeInstanceService.addDeletedAttributeInstance( + any(), + NGSILD_NULL, + deletedAt, + expandAttribute( + INCOMING_PROPERTY, + """ + { + "type": "Property", + "value": "urn:ngsi-ld:null" + } + """.trimIndent(), + NGSILD_TEST_CORE_CONTEXTS + ).second[0] + ) } entityAttributeService.getForEntityAndAttribute(beehiveTestDId, INCOMING_PROPERTY) - .shouldFail { assertInstanceOf(ResourceNotFoundException::class.java, it) } + .shouldSucceedWith { + assertEquals(deletedAt, it.deletedAt) + } } @Test - fun `it should delete references of all temporal attribute instances`() = runTest { + fun `it should flag and audit all instances of a deleted attribute`() = runTest { val rawEntity = loadSampleData("beehive_multi_instance_property.jsonld") coEvery { attributeInstanceService.create(any()) } returns Unit.right() - coEvery { attributeInstanceService.deleteAllInstancesOfAttribute(any(), any()) } returns Unit.right() + coEvery { + attributeInstanceService.addDeletedAttributeInstance(any(), any(), any(), any()) + } returns Unit.right() entityAttributeService.createAttributes(rawEntity, APIC_COMPOUND_CONTEXTS) @@ -616,17 +782,88 @@ class EntityAttributeServiceTests : WithTimescaleContainer, WithKafkaContainer { beehiveTestCId, INCOMING_PROPERTY, null, - deleteAll = true + deleteAll = true, + ngsiLdDateTime() + ).shouldSucceed() + + coVerify(exactly = 2) { + attributeInstanceService.addDeletedAttributeInstance(any(), any(), any(), any()) + } + + entityAttributeService.getForEntityAndAttribute(beehiveTestCId, INCOMING_PROPERTY) + .shouldFail { assertInstanceOf(ResourceNotFoundException::class.java, it) } + } + + @Test + fun `it should flag and audit all deleted attributes of an entity`() = runTest { + val rawEntity = loadSampleData("beehive.jsonld") + + coEvery { attributeInstanceService.create(any()) } returns Unit.right() + coEvery { + attributeInstanceService.addDeletedAttributeInstance(any(), any(), any(), any()) + } returns Unit.right() + + entityAttributeService.createAttributes(rawEntity, APIC_COMPOUND_CONTEXTS) + + entityAttributeService.deleteAttributes( + beehiveTestCId, + ngsiLdDateTime() + ).shouldSucceed() + + coVerify(exactly = 4) { + attributeInstanceService.addDeletedAttributeInstance(any(), any(), any(), any()) + } + + entityAttributeService.getForEntityAndAttribute(beehiveTestCId, INCOMING_PROPERTY) + .shouldSucceedWith { assertTrue(it.deletedAt != null) } + } + + @Test + fun `it should permanently delete an attribute of an entity`() = runTest { + val rawEntity = loadSampleData("beehive.jsonld") + + coEvery { attributeInstanceService.create(any()) } returns Unit.right() + coEvery { attributeInstanceService.deleteInstancesOfAttribute(any(), any(), any()) } returns Unit.right() + + entityAttributeService.createAttributes(rawEntity, APIC_COMPOUND_CONTEXTS) + + entityAttributeService.permanentlyDeleteAttribute( + beehiveTestCId, + INCOMING_PROPERTY, + null, + false ).shouldSucceed() coVerify { - attributeInstanceService.deleteAllInstancesOfAttribute(eq(beehiveTestCId), eq(INCOMING_PROPERTY)) + attributeInstanceService.deleteInstancesOfAttribute(beehiveTestCId, INCOMING_PROPERTY, null) } entityAttributeService.getForEntityAndAttribute(beehiveTestCId, INCOMING_PROPERTY) .shouldFail { assertInstanceOf(ResourceNotFoundException::class.java, it) } } + @Test + fun `it should permanently delete all attribute of an entity`() = runTest { + val rawEntity = loadSampleData("beehive.jsonld") + + coEvery { attributeInstanceService.create(any()) } returns Unit.right() + coEvery { attributeInstanceService.deleteInstancesOfEntity(any()) } returns Unit.right() + + entityAttributeService.createAttributes(rawEntity, APIC_COMPOUND_CONTEXTS) + + entityAttributeService.permanentlyDeleteAttributes(beehiveTestCId).shouldSucceed() + + coVerify { + attributeInstanceService.deleteInstancesOfEntity( + match { it.size == 4 } + ) + } + + entityAttributeService.getForEntity(beehiveTestCId, emptySet(), emptySet()).also { + assertTrue(it.isEmpty()) + } + } + @Test fun `it should return a right unit if entiy and attribute exist`() = runTest { val rawEntity = loadSampleData() diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityEventServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityEventServiceTests.kt index dd2914563..907eabb63 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityEventServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityEventServiceTests.kt @@ -2,12 +2,10 @@ package com.egm.stellio.search.entity.service import arrow.core.right import com.egm.stellio.search.entity.model.Entity -import com.egm.stellio.search.entity.model.UpdateOperationResult -import com.egm.stellio.search.entity.model.UpdateResult -import com.egm.stellio.search.entity.model.UpdatedDetails +import com.egm.stellio.search.entity.model.OperationStatus +import com.egm.stellio.search.entity.model.SucceededAttributeOperationResult import com.egm.stellio.search.support.EMPTY_PAYLOAD import com.egm.stellio.shared.model.AttributeAppendEvent -import com.egm.stellio.shared.model.AttributeDeleteAllInstancesEvent import com.egm.stellio.shared.model.AttributeDeleteEvent import com.egm.stellio.shared.model.AttributeReplaceEvent import com.egm.stellio.shared.model.AttributeUpdateEvent @@ -16,8 +14,8 @@ import com.egm.stellio.shared.model.EntityEvent import com.egm.stellio.shared.model.EventsType import com.egm.stellio.shared.model.ExpandedAttribute import com.egm.stellio.shared.model.ExpandedAttributeInstance +import com.egm.stellio.shared.model.ExpandedEntity import com.egm.stellio.shared.model.ExpandedTerm -import com.egm.stellio.shared.model.toExpandedAttributes import com.egm.stellio.shared.util.AQUAC_COMPOUND_CONTEXT import com.egm.stellio.shared.util.JsonLdUtils.expandAttribute import com.egm.stellio.shared.util.JsonLdUtils.expandAttributes @@ -32,6 +30,7 @@ import io.mockk.coVerify import io.mockk.every import io.mockk.mockk import io.mockk.verify +import io.r2dbc.postgresql.codec.Json import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Test import org.springframework.boot.test.context.SpringBootTest @@ -135,7 +134,7 @@ class EntityEventServiceTests { } every { kafkaTemplate.send(any(), any(), any()) } returns CompletableFuture() - entityEventService.publishEntityDeleteEvent(null, entity).join() + entityEventService.publishEntityDeleteEvent(null, entity, ExpandedEntity(emptyMap())).join() verify { kafkaTemplate.send("cim.entity._CatchAll", breedingServiceUri.toString(), any()) } } @@ -151,12 +150,13 @@ class EntityEventServiceTests { entityEventService.publishAttributeChangeEvents( "sub", breedingServiceUri, - expandedAttribute.toExpandedAttributes(), - UpdateResult( - listOf(UpdatedDetails(fishNumberProperty, null, UpdateOperationResult.APPENDED)), - emptyList() - ), - true + listOf( + SucceededAttributeOperationResult( + attributeName = fishNumberProperty, + operationStatus = OperationStatus.APPENDED, + newExpandedValue = expandedAttribute.second[0] + ) + ) ).join() verify { @@ -186,12 +186,13 @@ class EntityEventServiceTests { entityEventService.publishAttributeChangeEvents( null, breedingServiceUri, - expandedAttribute.toExpandedAttributes(), - UpdateResult( - listOf(UpdatedDetails(fishNumberProperty, null, UpdateOperationResult.REPLACED)), - emptyList() - ), - true + listOf( + SucceededAttributeOperationResult( + attributeName = fishNumberProperty, + operationStatus = OperationStatus.REPLACED, + newExpandedValue = expandedAttribute.second[0] + ) + ) ).join() verify { @@ -222,24 +223,24 @@ class EntityEventServiceTests { } """.trimIndent() val jsonLdAttributes = expandAttributes(attributesPayload, listOf(AQUAC_COMPOUND_CONTEXT)) - val appendResult = UpdateResult( - listOf( - UpdatedDetails(fishNumberProperty, null, UpdateOperationResult.APPENDED), - UpdatedDetails(fishNameProperty, fishName1DatasetUri, UpdateOperationResult.REPLACED) + val operationResult = listOf( + SucceededAttributeOperationResult( + attributeName = fishNumberProperty, + operationStatus = OperationStatus.APPENDED, + newExpandedValue = jsonLdAttributes[fishNumberProperty]!![0] ), - emptyList() + SucceededAttributeOperationResult( + attributeName = fishNameProperty, + datasetId = fishName1DatasetUri, + operationStatus = OperationStatus.REPLACED, + newExpandedValue = jsonLdAttributes[fishNameProperty]!![0] + ) ) coEvery { entityQueryService.retrieve(breedingServiceUri) } returns entity.right() every { entity.types } returns listOf(breedingServiceType) - entityEventService.publishAttributeChangeEvents( - null, - breedingServiceUri, - jsonLdAttributes, - appendResult, - true - ).join() + entityEventService.publishAttributeChangeEvents(null, breedingServiceUri, operationResult).join() verify { entityEventService["publishEntityEvent"]( @@ -287,12 +288,18 @@ class EntityEventServiceTests { } """.trimIndent() val jsonLdAttributes = expandAttributes(attributesPayload, listOf(AQUAC_COMPOUND_CONTEXT)) - val updateResult = UpdateResult( - updated = arrayListOf( - UpdatedDetails(fishNameProperty, fishName1DatasetUri, UpdateOperationResult.REPLACED), - UpdatedDetails(fishNumberProperty, null, UpdateOperationResult.REPLACED) + val operationResult = listOf( + SucceededAttributeOperationResult( + attributeName = fishNumberProperty, + operationStatus = OperationStatus.REPLACED, + newExpandedValue = jsonLdAttributes[fishNumberProperty]!![0] ), - notUpdated = arrayListOf() + SucceededAttributeOperationResult( + attributeName = fishNameProperty, + datasetId = fishName1DatasetUri, + operationStatus = OperationStatus.REPLACED, + newExpandedValue = jsonLdAttributes[fishNameProperty]!![0] + ) ) coEvery { entityQueryService.retrieve(breedingServiceUri) } returns entity.right() @@ -301,9 +308,7 @@ class EntityEventServiceTests { entityEventService.publishAttributeChangeEvents( null, breedingServiceUri, - jsonLdAttributes, - updateResult, - true + operationResult ).join() verify { @@ -346,24 +351,25 @@ class EntityEventServiceTests { } """.trimIndent() val jsonLdAttributes = expandAttributes(attributePayload, listOf(AQUAC_COMPOUND_CONTEXT)) - val updateResult = UpdateResult( - updated = arrayListOf( - UpdatedDetails(fishNameProperty, fishName1DatasetUri, UpdateOperationResult.REPLACED), - UpdatedDetails(fishNameProperty, fishName2DatasetUri, UpdateOperationResult.REPLACED) + val operationResult = listOf( + SucceededAttributeOperationResult( + attributeName = fishNameProperty, + datasetId = fishName1DatasetUri, + operationStatus = OperationStatus.REPLACED, + newExpandedValue = jsonLdAttributes[fishNameProperty]!![0] ), - notUpdated = arrayListOf() + SucceededAttributeOperationResult( + attributeName = fishNameProperty, + datasetId = fishName2DatasetUri, + operationStatus = OperationStatus.REPLACED, + newExpandedValue = jsonLdAttributes[fishNameProperty]!![1] + ) ) coEvery { entityQueryService.retrieve(breedingServiceUri) } returns entity.right() every { entity.types } returns listOf(breedingServiceType) - entityEventService.publishAttributeChangeEvents( - null, - breedingServiceUri, - jsonLdAttributes, - updateResult, - true - ).join() + entityEventService.publishAttributeChangeEvents(null, breedingServiceUri, operationResult).join() verify { entityEventService["publishEntityEvent"]( @@ -398,8 +404,13 @@ class EntityEventServiceTests { fishNameAttributeFragment, listOf(AQUAC_COMPOUND_CONTEXT) ) - val updatedDetails = listOf( - UpdatedDetails(fishNameProperty, fishName1DatasetUri, UpdateOperationResult.UPDATED) + val operationResult = listOf( + SucceededAttributeOperationResult( + attributeName = fishNameProperty, + datasetId = fishName1DatasetUri, + operationStatus = OperationStatus.UPDATED, + newExpandedValue = expandedAttribute.second[0] + ) ) coEvery { entityQueryService.retrieve(breedingServiceUri) } returns entity.right() @@ -408,9 +419,7 @@ class EntityEventServiceTests { entityEventService.publishAttributeChangeEvents( null, breedingServiceUri, - expandedAttribute.toExpandedAttributes(), - UpdateResult(updatedDetails, emptyList()), - false + operationResult ).join() verify { @@ -429,39 +438,47 @@ class EntityEventServiceTests { } @Test - fun `it should publish ATTRIBUTE_DELETE_ALL_INSTANCE event if all instances of an attribute are deleted`() = - runTest { - val entity = mockk(relaxed = true) - - coEvery { entityQueryService.retrieve(breedingServiceUri) } returns entity.right() - every { entity.types } returns listOf(breedingServiceType) + fun `it should publish ATTRIBUTE_DELETE event if an attribute has been deleted as part of an update `() = runTest { + val entity = mockk(relaxed = true).apply { + every { payload } returns Json.of("{}") + every { types } returns listOf(breedingServiceType) + } + coEvery { entityQueryService.retrieve(breedingServiceUri) } returns entity.right() - entityEventService.publishAttributeDeleteEvent( - null, - breedingServiceUri, - fishNameProperty, - null, - true - ).join() + entityEventService.publishAttributeChangeEvents( + null, + breedingServiceUri, + listOf( + SucceededAttributeOperationResult( + fishNameProperty, + fishName1DatasetUri, + OperationStatus.DELETED, + emptyMap() + ) + ) + ).join() - verify { - entityEventService["publishEntityEvent"]( - match { entityEvent -> - listOf(entityEvent).all { - it.operationType == EventsType.ATTRIBUTE_DELETE_ALL_INSTANCES && - it.entityId == breedingServiceUri && - it.entityTypes == listOf(breedingServiceType) && - it.attributeName == fishNameProperty && - it.contexts.isEmpty() - } + verify { + entityEventService["publishEntityEvent"]( + match { entityEvent -> + listOf(entityEvent).all { + it.operationType == EventsType.ATTRIBUTE_DELETE && + it.entityId == breedingServiceUri && + it.entityTypes == listOf(breedingServiceType) && + it.attributeName == fishNameProperty && + it.datasetId == fishName1DatasetUri && + it.contexts.isEmpty() } - ) - } + } + ) } + } @Test fun `it should publish ATTRIBUTE_DELETE event if an instance of an attribute is deleted`() = runTest { - val entity = mockk(relaxed = true) + val entity = mockk(relaxed = true).apply { + every { payload } returns Json.of("{}") + } coEvery { entityQueryService.retrieve(breedingServiceUri) } returns entity.right() every { entity.types } returns listOf(breedingServiceType) @@ -469,9 +486,12 @@ class EntityEventServiceTests { entityEventService.publishAttributeDeleteEvent( null, breedingServiceUri, - fishNameProperty, - fishName1DatasetUri, - false + SucceededAttributeOperationResult( + fishNameProperty, + fishName1DatasetUri, + OperationStatus.DELETED, + emptyMap() + ) ).join() verify { diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityOperationServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityOperationServiceTests.kt index 0cbb4a47d..004a15c6f 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityOperationServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityOperationServiceTests.kt @@ -296,7 +296,10 @@ class EntityOperationServiceTests { ) assertTrue(batchOperationResult.errors.isEmpty()) - coVerify { entityService.replaceEntity(firstEntityURI, any(), any(), any()) } + coVerify { + entityService.replaceEntity(firstEntityURI, any(), any(), any()) + entityService.replaceEntity(secondEntityURI, any(), any(), any()) + } } @Test diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityQueryServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityQueryServiceTests.kt index 527bf7d52..c60a3fd4e 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityQueryServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityQueryServiceTests.kt @@ -7,7 +7,6 @@ import com.egm.stellio.search.entity.model.Entity import com.egm.stellio.search.support.WithKafkaContainer import com.egm.stellio.search.support.WithTimescaleContainer import com.egm.stellio.search.support.buildDefaultQueryParams -import com.egm.stellio.shared.model.AlreadyExistsException import com.egm.stellio.shared.model.ResourceNotFoundException import com.egm.stellio.shared.util.AUTHZ_TEST_COMPOUND_CONTEXTS import com.egm.stellio.shared.util.AuthContextModel.SpecificAccessPolicy.AUTH_READ @@ -29,6 +28,7 @@ import kotlinx.coroutines.test.runTest import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired @@ -40,7 +40,7 @@ import java.time.ZonedDateTime @SpringBootTest @ActiveProfiles("test") -class EntityQueryServiceTests : WithTimescaleContainer, WithKafkaContainer { +class EntityQueryServiceTests : WithTimescaleContainer, WithKafkaContainer() { @Autowired private lateinit var entityQueryService: EntityQueryService @@ -96,14 +96,15 @@ class EntityQueryServiceTests : WithTimescaleContainer, WithKafkaContainer { @Test fun `it should return a list of JSON-LD entities when querying entities`() = runTest { + coEvery { authorizationService.userCanCreateEntities(any()) } returns Unit.right() + coEvery { authorizationService.createOwnerRight(any(), any()) } returns Unit.right() coEvery { authorizationService.computeAccessRightFilter(any()) } returns { null } loadAndPrepareSampleData("beehive.jsonld") .map { - entityService.createEntityPayload( + entityService.createEntity( it.second, - it.first, - now, + it.first ).shouldSucceed() } @@ -210,7 +211,7 @@ class EntityQueryServiceTests : WithTimescaleContainer, WithKafkaContainer { } @Test - fun `it should check the existence or non-existence of an entity`() = runTest { + fun `it should check the existence of an entity`() = runTest { loadMinimalEntity(entity01Uri, setOf(BEEHIVE_TYPE)) .sampleDataToNgsiLdEntity() .map { @@ -224,8 +225,25 @@ class EntityQueryServiceTests : WithTimescaleContainer, WithKafkaContainer { entityQueryService.checkEntityExistence(entity01Uri).shouldSucceed() entityQueryService.checkEntityExistence(entity02Uri) .shouldFail { assert(it is ResourceNotFoundException) } - entityQueryService.checkEntityExistence(entity01Uri, true) - .shouldFail { assert(it is AlreadyExistsException) } - entityQueryService.checkEntityExistence(entity02Uri, true).shouldSucceed() + } + + @Test + fun `it should check the state of an entity`() = runTest { + loadMinimalEntity(entity01Uri, setOf(BEEHIVE_TYPE)) + .sampleDataToNgsiLdEntity() + .map { + entityService.createEntityPayload( + it.second, + it.first, + now + ) + } + + entityQueryService.isMarkedAsDeleted(entity01Uri) + .shouldSucceedWith { + assertFalse(it) + } + entityQueryService.isMarkedAsDeleted(entity02Uri) + .shouldFail { assert(it is ResourceNotFoundException) } } } diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityServiceQueriesTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityServiceQueriesTests.kt index 4b6396bd9..f8c69a2aa 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityServiceQueriesTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityServiceQueriesTests.kt @@ -2,6 +2,7 @@ package com.egm.stellio.search.entity.service import arrow.core.right import com.egm.stellio.search.authorization.service.AuthorizationService +import com.egm.stellio.search.entity.model.Attribute import com.egm.stellio.search.entity.model.EntitiesQueryFromGet import com.egm.stellio.search.entity.model.EntitiesQueryFromPost import com.egm.stellio.search.entity.model.Entity @@ -42,6 +43,7 @@ import org.junit.jupiter.params.provider.CsvSource import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest import org.springframework.data.r2dbc.core.R2dbcEntityTemplate +import org.springframework.data.r2dbc.core.delete import org.springframework.data.relational.core.query.Criteria import org.springframework.data.relational.core.query.Query import org.springframework.data.relational.core.query.Update @@ -50,7 +52,7 @@ import java.net.URI @SpringBootTest @ActiveProfiles("test") -class EntityServiceQueriesTests : WithTimescaleContainer, WithKafkaContainer { +class EntityServiceQueriesTests : WithTimescaleContainer, WithKafkaContainer() { @Autowired private lateinit var entityQueryService: EntityQueryService @@ -95,15 +97,8 @@ class EntityServiceQueriesTests : WithTimescaleContainer, WithKafkaContainer { @AfterAll fun deleteEntities() { - coEvery { authorizationService.userCanAdminEntity(any(), any()) } returns Unit.right() - coEvery { authorizationService.removeRightsOnEntity(any()) } returns Unit.right() - runBlocking { - entityService.deleteEntity(entity01Uri) - entityService.deleteEntity(entity02Uri) - entityService.deleteEntity(entity03Uri) - entityService.deleteEntity(entity04Uri) - entityService.deleteEntity(entity05Uri) - } + r2dbcEntityTemplate.delete().from("entity_payload").all().block() + r2dbcEntityTemplate.delete().from("temporal_entity_attribute").all().block() } @ParameterizedTest diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityServiceTests.kt index 5a5b03897..5c079f2b8 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/entity/service/EntityServiceTests.kt @@ -1,58 +1,65 @@ package com.egm.stellio.search.entity.service +import arrow.core.left import arrow.core.right import arrow.core.toOption import com.egm.stellio.search.authorization.service.AuthorizationService import com.egm.stellio.search.common.util.deserializeAsMap -import com.egm.stellio.search.entity.model.EMPTY_UPDATE_RESULT import com.egm.stellio.search.entity.model.Entity -import com.egm.stellio.search.entity.model.UpdateOperationResult -import com.egm.stellio.search.entity.model.UpdateResult -import com.egm.stellio.search.entity.model.UpdatedDetails -import com.egm.stellio.search.support.EMPTY_PAYLOAD +import com.egm.stellio.search.entity.model.OperationStatus +import com.egm.stellio.search.entity.model.SucceededAttributeOperationResult import com.egm.stellio.search.support.WithKafkaContainer import com.egm.stellio.search.support.WithTimescaleContainer +import com.egm.stellio.shared.model.AccessDeniedException +import com.egm.stellio.shared.model.AlreadyExistsException +import com.egm.stellio.shared.model.ResourceNotFoundException import com.egm.stellio.shared.util.APIARY_TYPE import com.egm.stellio.shared.util.APIC_COMPOUND_CONTEXTS import com.egm.stellio.shared.util.BEEHIVE_TYPE import com.egm.stellio.shared.util.INCOMING_PROPERTY +import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_ID import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DEFAULT_VOCAB +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DELETED_AT_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_SCOPE_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.expandAttribute import com.egm.stellio.shared.util.JsonLdUtils.expandAttributes import com.egm.stellio.shared.util.JsonUtils.deserializeExpandedPayload +import com.egm.stellio.shared.util.OUTGOING_PROPERTY +import com.egm.stellio.shared.util.loadAndExpandDeletedEntity import com.egm.stellio.shared.util.loadAndPrepareSampleData import com.egm.stellio.shared.util.loadMinimalEntity import com.egm.stellio.shared.util.loadSampleData import com.egm.stellio.shared.util.ngsiLdDateTime import com.egm.stellio.shared.util.sampleDataToNgsiLdEntity +import com.egm.stellio.shared.util.shouldFail import com.egm.stellio.shared.util.shouldSucceed import com.egm.stellio.shared.util.shouldSucceedAndResult import com.egm.stellio.shared.util.shouldSucceedWith import com.egm.stellio.shared.util.toUri import com.ninjasquad.springmockk.MockkBean +import io.mockk.Called import io.mockk.coEvery import io.mockk.coVerify import kotlinx.coroutines.Job import kotlinx.coroutines.test.runTest +import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertInstanceOf import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test -import org.junit.jupiter.api.assertThrows import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest -import org.springframework.dao.DataIntegrityViolationException import org.springframework.data.r2dbc.core.R2dbcEntityTemplate import org.springframework.data.r2dbc.core.delete import org.springframework.test.context.ActiveProfiles @SpringBootTest @ActiveProfiles("test") -class EntityServiceTests : WithTimescaleContainer, WithKafkaContainer { +class EntityServiceTests : WithTimescaleContainer, WithKafkaContainer() { @Autowired private lateinit var entityService: EntityService @@ -145,7 +152,7 @@ class EntityServiceTests : WithTimescaleContainer, WithKafkaContainer { } @Test - fun `it should not create an entity payload if one already existed`() = runTest { + fun `it should not create an entity payload if one already exists`() = runTest { val (jsonLdEntity, ngsiLdEntity) = loadMinimalEntity(entity01Uri, setOf(BEEHIVE_TYPE)).sampleDataToNgsiLdEntity().shouldSucceedAndResult() entityService.createEntityPayload( @@ -154,13 +161,92 @@ class EntityServiceTests : WithTimescaleContainer, WithKafkaContainer { now, ) - assertThrows { - entityService.createEntityPayload( - ngsiLdEntity, - jsonLdEntity, - now, - ) - } + entityService.createEntity( + ngsiLdEntity, + jsonLdEntity, + sub, + ).shouldFail { assertInstanceOf(AlreadyExistsException::class.java, it) } + } + + @Test + fun `it should allow to create an entity over a deleted one if authorized`() = runTest { + coEvery { entityAttributeService.deleteAttributes(any(), any()) } returns Unit.right() + coEvery { authorizationService.userCanAdminEntity(any(), any()) } returns Unit.right() + coEvery { entityAttributeService.createAttributes(any(), any(), any(), any(), any()) } returns Unit.right() + coEvery { authorizationService.createOwnerRight(any(), any()) } returns Unit.right() + + val (expandedEntity, ngsiLdEntity) = + loadMinimalEntity(entity01Uri, setOf(BEEHIVE_TYPE)) + .sampleDataToNgsiLdEntity() + .shouldSucceedAndResult() + + entityService.createEntityPayload( + ngsiLdEntity, + expandedEntity, + now + ) + + entityService.deleteEntityPayload(entity01Uri, ngsiLdDateTime(), loadAndExpandDeletedEntity(entity01Uri)) + .shouldSucceedWith { + assertEquals(entity01Uri, it.entityId) + assertNotNull(it.payload) + } + + entityService.createEntity(ngsiLdEntity, expandedEntity, null) + .shouldSucceed() + } + + @Test + fun `it should not allow to create an entity over a deleted one if not authorized`() = runTest { + coEvery { entityAttributeService.deleteAttributes(any(), any()) } returns Unit.right() + coEvery { + authorizationService.userCanAdminEntity(any(), any()) + } returns AccessDeniedException("Unauthorized").left() + + val (expandedEntity, ngsiLdEntity) = + loadMinimalEntity(entity01Uri, setOf(BEEHIVE_TYPE)) + .sampleDataToNgsiLdEntity() + .shouldSucceedAndResult() + + entityService.createEntityPayload( + ngsiLdEntity, + expandedEntity, + now + ) + + entityService.deleteEntityPayload(entity01Uri, ngsiLdDateTime(), loadAndExpandDeletedEntity(entity01Uri)) + .shouldSucceedWith { + assertEquals(entity01Uri, it.entityId) + assertNotNull(it.payload) + } + + entityService.createEntity(ngsiLdEntity, expandedEntity, null) + .shouldFail { assertInstanceOf(AccessDeniedException::class.java, it) } + } + + @Test + fun `it should create the deleted representation of an entity when deleting it`() = runTest { + val (expandedEntity, ngsiLdEntity) = + loadMinimalEntity(entity01Uri, setOf(BEEHIVE_TYPE)) + .sampleDataToNgsiLdEntity() + .shouldSucceedAndResult() + + entityService.createEntityPayload( + ngsiLdEntity, + expandedEntity, + now + ).shouldSucceed() + + entityService.deleteEntityPayload(entity01Uri, ngsiLdDateTime(), loadAndExpandDeletedEntity(entity01Uri)) + .shouldSucceed() + + entityQueryService.retrieve(entity01Uri, true) + .shouldSucceedWith { entity -> + val payload = entity.payload.deserializeAsMap() + assertThat(payload) + .hasSize(2) + .containsKeys(JSONLD_ID, NGSILD_DELETED_AT_PROPERTY) + } } @Test @@ -172,9 +258,8 @@ class EntityServiceTests : WithTimescaleContainer, WithKafkaContainer { } returns Unit.right() coEvery { entityAttributeService.mergeAttributes(any(), any(), any(), any(), any(), any()) - } returns UpdateResult( - listOf(UpdatedDetails(INCOMING_PROPERTY, null, UpdateOperationResult.APPENDED)), - emptyList() + } returns listOf( + SucceededAttributeOperationResult(INCOMING_PROPERTY, null, OperationStatus.APPENDED, emptyMap()), ).right() coEvery { entityAttributeService.getForEntity(any(), any(), any()) } returns emptyList() coEvery { authorizationService.createOwnerRight(any(), any()) } returns Unit.right() @@ -238,9 +323,8 @@ class EntityServiceTests : WithTimescaleContainer, WithKafkaContainer { } returns Unit.right() coEvery { entityAttributeService.mergeAttributes(any(), any(), any(), any(), any(), any()) - } returns UpdateResult( - listOf(UpdatedDetails(INCOMING_PROPERTY, null, UpdateOperationResult.APPENDED)), - emptyList() + } returns listOf( + SucceededAttributeOperationResult(INCOMING_PROPERTY, null, OperationStatus.APPENDED, emptyMap()) ).right() coEvery { entityAttributeService.getForEntity(any(), any(), any()) } returns emptyList() coEvery { authorizationService.createOwnerRight(any(), any()) } returns Unit.right() @@ -281,13 +365,9 @@ class EntityServiceTests : WithTimescaleContainer, WithKafkaContainer { } returns Unit.right() coEvery { entityAttributeService.mergeAttributes(any(), any(), any(), any(), any(), any()) - } returns UpdateResult( - listOf(UpdatedDetails(INCOMING_PROPERTY, null, UpdateOperationResult.APPENDED)), - emptyList() + } returns listOf( + SucceededAttributeOperationResult(INCOMING_PROPERTY, null, OperationStatus.APPENDED, emptyMap()) ).right() - coEvery { - entityAttributeService.partialUpdateAttribute(any(), any(), any(), any()) - } returns EMPTY_UPDATE_RESULT.right() coEvery { entityAttributeService.getForEntity(any(), any(), any()) } returns emptyList() coEvery { authorizationService.createOwnerRight(any(), any()) } returns Unit.right() @@ -334,7 +414,7 @@ class EntityServiceTests : WithTimescaleContainer, WithKafkaContainer { coEvery { entityAttributeService.createAttributes(any(), any(), any(), any(), any()) } returns Unit.right() - coEvery { entityAttributeService.deleteAttributes(any()) } returns Unit.right() + coEvery { entityAttributeService.deleteAttributes(any(), any()) } returns Unit.right() coEvery { authorizationService.createOwnerRight(any(), any()) } returns Unit.right() val (expandedEntity, ngsiLdEntity) = @@ -364,7 +444,7 @@ class EntityServiceTests : WithTimescaleContainer, WithKafkaContainer { coVerify { authorizationService.userCanCreateEntities(sub.toOption()) authorizationService.userCanUpdateEntity(beehiveTestCId, sub.toOption()) - entityAttributeService.deleteAttributes(beehiveTestCId) + entityAttributeService.deleteAttributes(beehiveTestCId, any()) entityAttributeService.createAttributes( any(), any(), @@ -382,9 +462,10 @@ class EntityServiceTests : WithTimescaleContainer, WithKafkaContainer { coEvery { entityAttributeService.getForEntity(any(), any(), any()) } returns emptyList() coEvery { entityAttributeService.replaceAttribute(any(), any(), any(), any(), any()) - } returns UpdateResult( - updated = listOf(UpdatedDetails(INCOMING_PROPERTY, null, UpdateOperationResult.REPLACED)), - notUpdated = emptyList() + } returns SucceededAttributeOperationResult( + attributeName = INCOMING_PROPERTY, + operationStatus = OperationStatus.REPLACED, + newExpandedValue = emptyMap() ).right() val (jsonLdEntity, ngsiLdEntity) = loadSampleData().sampleDataToNgsiLdEntity().shouldSucceedAndResult() @@ -399,8 +480,7 @@ class EntityServiceTests : WithTimescaleContainer, WithKafkaContainer { .shouldSucceedWith { it.updated.size == 1 && it.notUpdated.isEmpty() && - it.updated[0].attributeName == INCOMING_PROPERTY && - it.updated[0].updateOperationResult == UpdateOperationResult.REPLACED + it.updated[0] == INCOMING_PROPERTY } } @@ -417,11 +497,9 @@ class EntityServiceTests : WithTimescaleContainer, WithKafkaContainer { entityService.updateTypes(beehiveTestCId, listOf(BEEHIVE_TYPE, APIARY_TYPE), ngsiLdDateTime(), false) .shouldSucceedWith { - assertTrue(it.isSuccessful()) - assertEquals(1, it.updated.size) - val updatedDetails = it.updated[0] - assertEquals(JSONLD_TYPE, updatedDetails.attributeName) - assertEquals(UpdateOperationResult.APPENDED, updatedDetails.updateOperationResult) + assertInstanceOf(SucceededAttributeOperationResult::class.java, it) + assertEquals(JSONLD_TYPE, it.attributeName) + assertEquals(OperationStatus.APPENDED, it.operationStatus) } entityQueryService.retrieve(beehiveTestCId) @@ -459,8 +537,16 @@ class EntityServiceTests : WithTimescaleContainer, WithKafkaContainer { } @Test - fun `it should upsert an entity payload if one already existed`() = runTest { - loadMinimalEntity(entity01Uri, setOf(BEEHIVE_TYPE)) + fun `it should remove the scopes from an entity`() = runTest { + coEvery { authorizationService.userCanUpdateEntity(any(), any()) } returns Unit.right() + coEvery { + entityAttributeService.addAttribute(any(), any(), any(), any(), any(), any()) + } returns Unit.right() + coEvery { + entityAttributeService.getForEntity(any(), any(), any()) + } returns emptyList() + + loadSampleData("beehive_with_scope.jsonld") .sampleDataToNgsiLdEntity() .map { entityService.createEntityPayload( @@ -470,13 +556,20 @@ class EntityServiceTests : WithTimescaleContainer, WithKafkaContainer { ) } - entityService.upsertEntityPayload(entity01Uri, EMPTY_PAYLOAD) + entityService.deleteAttribute(beehiveTestCId, NGSILD_SCOPE_PROPERTY, null) .shouldSucceed() + + entityQueryService.retrieve(beehiveTestCId) + .shouldSucceedWith { + assertNull(it.scopes) + } } @Test - fun `it should delete an entity payload`() = runTest { - coEvery { entityAttributeService.deleteAttributes(any()) } returns Unit.right() + fun `it should permanently delete an entity`() = runTest { + coEvery { authorizationService.userCanAdminEntity(any(), any()) } returns Unit.right() + coEvery { entityAttributeService.permanentlyDeleteAttributes(any()) } returns Unit.right() + coEvery { authorizationService.removeRightsOnEntity(any()) } returns Unit.right() loadMinimalEntity(entity01Uri, setOf(BEEHIVE_TYPE)) .sampleDataToNgsiLdEntity() @@ -488,36 +581,47 @@ class EntityServiceTests : WithTimescaleContainer, WithKafkaContainer { ) } - entityService.deleteEntityPayload(entity01Uri) - .shouldSucceedWith { - assertEquals(entity01Uri, it.entityId) - assertNotNull(it.payload) + entityService.permanentlyDeleteEntity(entity01Uri).shouldSucceed() + + entityQueryService.retrieve(entity01Uri) + .shouldFail { + assertInstanceOf(ResourceNotFoundException::class.java, it) } + } - // if correctly deleted, we should be able to create a new one - loadMinimalEntity(entity01Uri, setOf(BEEHIVE_TYPE)) - .sampleDataToNgsiLdEntity() + @Test + fun `it should permanently delete an attribute`() = runTest { + coEvery { authorizationService.userCanUpdateEntity(any(), any()) } returns Unit.right() + coEvery { entityAttributeService.checkEntityAndAttributeExistence(any(), any(), any()) } returns Unit.right() + coEvery { entityAttributeService.permanentlyDeleteAttribute(any(), any(), any(), any()) } returns Unit.right() + coEvery { entityAttributeService.getForEntity(any(), any(), any()) } returns emptyList() + + loadAndPrepareSampleData("beehive.jsonld") .map { entityService.createEntityPayload( it.second, it.first, now - ).shouldSucceed() + ) } + + entityService.permanentlyDeleteAttribute(beehiveTestCId, INCOMING_PROPERTY, null).shouldSucceed() + + coVerify { + entityAttributeService.checkEntityAndAttributeExistence(beehiveTestCId, INCOMING_PROPERTY, null) + entityAttributeService.permanentlyDeleteAttribute(beehiveTestCId, INCOMING_PROPERTY, null, false) + entityAttributeService.getForEntity(beehiveTestCId, emptySet(), emptySet()) + } } @Test - fun `it should remove the scopes from an entity`() = runTest { + fun `it should return a ResourceNotFound error if trying to permanently delete an unknown attribute`() = runTest { coEvery { authorizationService.userCanUpdateEntity(any(), any()) } returns Unit.right() coEvery { - entityAttributeService.addAttribute(any(), any(), any(), any(), any(), any()) - } returns Unit.right() - coEvery { - entityAttributeService.getForEntity(any(), any(), any()) - } returns emptyList() + entityAttributeService.checkEntityAndAttributeExistence(any(), any(), any()) + } returns ResourceNotFoundException("Entity does not exist").left() - loadSampleData("beehive_with_scope.jsonld") - .sampleDataToNgsiLdEntity() + loadAndPrepareSampleData("beehive.jsonld") .map { entityService.createEntityPayload( it.second, @@ -526,12 +630,16 @@ class EntityServiceTests : WithTimescaleContainer, WithKafkaContainer { ) } - entityService.deleteAttribute(beehiveTestCId, NGSILD_SCOPE_PROPERTY, null) - .shouldSucceed() + entityService.permanentlyDeleteAttribute(beehiveTestCId, OUTGOING_PROPERTY, null).shouldFail { + assertInstanceOf(ResourceNotFoundException::class.java, it) + } - entityQueryService.retrieve(beehiveTestCId) - .shouldSucceedWith { - assertNull(it.scopes) - } + coVerify { + entityAttributeService.checkEntityAndAttributeExistence(beehiveTestCId, OUTGOING_PROPERTY, null) + listOf( + entityAttributeService.permanentlyDeleteAttribute(beehiveTestCId, OUTGOING_PROPERTY, null, false), + entityAttributeService.getForEntity(beehiveTestCId, emptySet(), emptySet()) + ) wasNot Called + } } } diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/entity/util/AttributeUtilsTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/entity/util/AttributeUtilsTests.kt index b9d039538..5c15c6ad4 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/entity/util/AttributeUtilsTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/entity/util/AttributeUtilsTests.kt @@ -1,11 +1,13 @@ package com.egm.stellio.search.entity.util import com.egm.stellio.search.entity.model.Attribute +import com.egm.stellio.search.entity.model.Attribute.AttributeType import com.egm.stellio.shared.util.JsonLdUtils.expandAttribute import com.egm.stellio.shared.util.NGSILD_TEST_CORE_CONTEXTS import com.egm.stellio.shared.util.ngsiLdDateTime import kotlinx.coroutines.test.runTest import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import org.springframework.test.context.ActiveProfiles import java.net.URI @@ -23,7 +25,7 @@ class AttributeUtilsTests { ) assertEquals( Attribute.AttributeValueType.STRING, - guessAttributeValueType(Attribute.AttributeType.Property, expandedStringProperty.second[0]) + guessAttributeValueType(AttributeType.Property, expandedStringProperty.second[0]) ) } @@ -36,7 +38,7 @@ class AttributeUtilsTests { ) assertEquals( Attribute.AttributeValueType.NUMBER, - guessAttributeValueType(Attribute.AttributeType.Property, expandedBooleanProperty.second[0]) + guessAttributeValueType(AttributeType.Property, expandedBooleanProperty.second[0]) ) } @@ -49,7 +51,7 @@ class AttributeUtilsTests { ) assertEquals( Attribute.AttributeValueType.NUMBER, - guessAttributeValueType(Attribute.AttributeType.Property, expandedBooleanProperty.second[0]) + guessAttributeValueType(AttributeType.Property, expandedBooleanProperty.second[0]) ) } @@ -62,7 +64,7 @@ class AttributeUtilsTests { ) assertEquals( Attribute.AttributeValueType.BOOLEAN, - guessAttributeValueType(Attribute.AttributeType.Property, expandedBooleanProperty.second[0]) + guessAttributeValueType(AttributeType.Property, expandedBooleanProperty.second[0]) ) } @@ -75,7 +77,7 @@ class AttributeUtilsTests { ) assertEquals( Attribute.AttributeValueType.OBJECT, - guessAttributeValueType(Attribute.AttributeType.Property, expandedListProperty.second[0]) + guessAttributeValueType(AttributeType.Property, expandedListProperty.second[0]) ) } @@ -88,7 +90,7 @@ class AttributeUtilsTests { ) assertEquals( Attribute.AttributeValueType.ARRAY, - guessAttributeValueType(Attribute.AttributeType.Property, expandedListProperty.second[0]) + guessAttributeValueType(AttributeType.Property, expandedListProperty.second[0]) ) } @@ -101,7 +103,7 @@ class AttributeUtilsTests { ) assertEquals( Attribute.AttributeValueType.TIME, - guessAttributeValueType(Attribute.AttributeType.Property, expandedTimeProperty.second[0]) + guessAttributeValueType(AttributeType.Property, expandedTimeProperty.second[0]) ) } @@ -114,7 +116,7 @@ class AttributeUtilsTests { ) assertEquals( Attribute.AttributeValueType.DATETIME, - guessAttributeValueType(Attribute.AttributeType.Property, expandedTimeProperty.second[0]) + guessAttributeValueType(AttributeType.Property, expandedTimeProperty.second[0]) ) } @@ -130,7 +132,7 @@ class AttributeUtilsTests { ) assertEquals( Attribute.AttributeValueType.GEOMETRY, - guessAttributeValueType(Attribute.AttributeType.GeoProperty, expandedGeoProperty.second[0]) + guessAttributeValueType(AttributeType.GeoProperty, expandedGeoProperty.second[0]) ) } @@ -146,7 +148,7 @@ class AttributeUtilsTests { ) assertEquals( Attribute.AttributeValueType.JSON, - guessAttributeValueType(Attribute.AttributeType.JsonProperty, expandedJsonProperty.second[0]) + guessAttributeValueType(AttributeType.JsonProperty, expandedJsonProperty.second[0]) ) } @@ -160,9 +162,79 @@ class AttributeUtilsTests { assertEquals( Attribute.AttributeValueType.URI, guessAttributeValueType( - Attribute.AttributeType.Relationship, + AttributeType.Relationship, expandedGeoRelationship.second[0] ) ) } + + @Test + fun `it should find a Property whose value is NGSI-LD Null`() = runTest { + val expandedProperty = expandAttribute( + """ + { + "property": { + "type": "Property", + "value": "urn:ngsi-ld:null" + } + } + """.trimIndent(), + NGSILD_TEST_CORE_CONTEXTS + ).second[0] + + assertTrue(hasNgsiLdNullValue(expandedProperty, AttributeType.Property)) + } + + @Test + fun `it should find a LanguageProperty whose value is NGSI-LD Null`() = runTest { + val expandedProperty = expandAttribute( + """ + { + "langProperty": { + "type": "LanguageProperty", + "languageMap": { + "@none": "urn:ngsi-ld:null" + } + } + } + """.trimIndent(), + NGSILD_TEST_CORE_CONTEXTS + ).second[0] + + assertTrue(hasNgsiLdNullValue(expandedProperty, AttributeType.LanguageProperty)) + } + + @Test + fun `it should find a JsonProperty whose value is NGSI-LD Null`() = runTest { + val expandedProperty = expandAttribute( + """ + { + "jsonProperty": { + "type": "JsonProperty", + "json": "urn:ngsi-ld:null" + } + } + """.trimIndent(), + NGSILD_TEST_CORE_CONTEXTS + ).second[0] + + assertTrue(hasNgsiLdNullValue(expandedProperty, AttributeType.JsonProperty)) + } + + @Test + fun `it should find a Relationship whose value is NGSI-LD Null`() = runTest { + val expandedProperty = expandAttribute( + """ + { + "relationship": { + "type": "Relationship", + "object": "urn:ngsi-ld:null" + } + } + """.trimIndent(), + NGSILD_TEST_CORE_CONTEXTS + ).second[0] + + assertTrue(hasNgsiLdNullValue(expandedProperty, AttributeType.Relationship)) + } } diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/entity/web/EntityHandlerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/entity/web/EntityHandlerTests.kt index aefc5d192..3b89c8169 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/entity/web/EntityHandlerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/entity/web/EntityHandlerTests.kt @@ -10,9 +10,7 @@ import com.egm.stellio.search.csr.service.ContextSourceCaller import com.egm.stellio.search.csr.service.ContextSourceRegistrationService import com.egm.stellio.search.entity.model.EntitiesQueryFromGet import com.egm.stellio.search.entity.model.NotUpdatedDetails -import com.egm.stellio.search.entity.model.UpdateOperationResult import com.egm.stellio.search.entity.model.UpdateResult -import com.egm.stellio.search.entity.model.UpdatedDetails import com.egm.stellio.search.entity.service.EntityQueryService import com.egm.stellio.search.entity.service.EntityService import com.egm.stellio.search.entity.service.LinkedEntityService @@ -1290,7 +1288,7 @@ class EntityHandlerTests { NGSILDWarning.HEADER_NAME, "199 urn:ngsi-ld:ContextSourceRegistration:test \"message with line breaks\"", "199 urn:ngsi-ld:ContextSourceRegistration:test \"message\"" - ).expectHeader().valueEquals(RESULTS_COUNT_HEADER, "0",) + ).expectHeader().valueEquals(RESULTS_COUNT_HEADER, "0") coVerify(exactly = 2) { contextSourceRegistrationService.updateContextSourceStatus(any(), false) } } @@ -1407,13 +1405,7 @@ class EntityHandlerTests { val jsonLdFile = ClassPathResource("/ngsild/aquac/fragments/BreedingService_newProperty.json") val entityId = "urn:ngsi-ld:BreedingService:0214".toUri() val appendResult = UpdateResult( - listOf( - UpdatedDetails( - fishNumberAttribute, - null, - UpdateOperationResult.APPENDED - ) - ), + listOf(fishNumberAttribute), emptyList() ) @@ -1444,13 +1436,7 @@ class EntityHandlerTests { val jsonLdFile = ClassPathResource("/ngsild/aquac/fragments/BreedingService_twoNewProperties.json") val entityId = "urn:ngsi-ld:BreedingService:0214".toUri() val appendResult = UpdateResult( - listOf( - UpdatedDetails( - fishNumberAttribute, - null, - UpdateOperationResult.APPENDED - ) - ), + listOf(fishNumberAttribute), listOf(NotUpdatedDetails(fishSizeAttribute, "overwrite disallowed")) ) @@ -1489,7 +1475,7 @@ class EntityHandlerTests { val jsonLdFile = ClassPathResource("/ngsild/aquac/fragments/BreedingService_newType.json") val entityId = "urn:ngsi-ld:BreedingService:0214".toUri() val appendTypeResult = UpdateResult( - listOf(UpdatedDetails(JSONLD_TYPE, null, UpdateOperationResult.APPENDED)), + listOf(JSONLD_TYPE), emptyList() ) @@ -1519,18 +1505,16 @@ class EntityHandlerTests { fun `append entity attribute should return a 207 if types or attributes could not be appended`() { val jsonLdFile = ClassPathResource("/ngsild/aquac/fragments/BreedingService_newInvalidTypeAndAttribute.json") val entityId = "urn:ngsi-ld:BreedingService:0214".toUri() - val appendTypeResult = UpdateResult( - emptyList(), - listOf(NotUpdatedDetails(JSONLD_TYPE, "Append operation has unexpectedly failed")) - ) - val appendResult = UpdateResult( - listOf(UpdatedDetails(fishNumberAttribute, null, UpdateOperationResult.APPENDED)), - listOf(NotUpdatedDetails(fishSizeAttribute, "overwrite disallowed")) - ) coEvery { entityService.appendAttributes(any(), any(), any(), any()) - } returns appendTypeResult.mergeWith(appendResult).right() + } returns UpdateResult( + updated = listOf(fishNumberAttribute), + notUpdated = listOf( + NotUpdatedDetails(JSONLD_TYPE, "Append operation has unexpectedly failed"), + NotUpdatedDetails(fishSizeAttribute, "overwrite disallowed") + ) + ).right() webClient.post() .uri("/ngsi-ld/v1/entities/$entityId/attrs") @@ -1649,10 +1633,8 @@ class EntityHandlerTests { val entityId = "urn:ngsi-ld:DeadFishes:019BN".toUri() val attrId = "fishNumber" val updateResult = UpdateResult( - updated = arrayListOf( - UpdatedDetails(fishNumberAttribute, "urn:ngsi-ld:Dataset:1".toUri(), UpdateOperationResult.UPDATED) - ), - notUpdated = arrayListOf() + updated = listOf(fishNumberAttribute), + notUpdated = emptyList() ) coEvery { @@ -1755,18 +1737,7 @@ class EntityHandlerTests { val jsonLdFile = ClassPathResource("/ngsild/aquac/fragments/DeadFishes_mergeEntity.json") val entityId = "urn:ngsi-ld:DeadFishes:019BN".toUri() val updateResult = UpdateResult( - updated = arrayListOf( - UpdatedDetails( - fishNumberAttribute, - null, - UpdateOperationResult.REPLACED - ), - UpdatedDetails( - fishSizeAttribute, - null, - UpdateOperationResult.APPENDED - ) - ), + updated = listOf(fishNumberAttribute, fishSizeAttribute), notUpdated = emptyList() ) @@ -1792,18 +1763,7 @@ class EntityHandlerTests { val jsonLdFile = ClassPathResource("/ngsild/aquac/fragments/DeadFishes_mergeEntity.json") val entityId = "urn:ngsi-ld:DeadFishes:019BN".toUri() val updateResult = UpdateResult( - updated = arrayListOf( - UpdatedDetails( - fishNumberAttribute, - null, - UpdateOperationResult.REPLACED - ), - UpdatedDetails( - fishSizeAttribute, - null, - UpdateOperationResult.APPENDED - ) - ), + updated = listOf(fishNumberAttribute, fishSizeAttribute), notUpdated = emptyList() ) @@ -1931,13 +1891,7 @@ class EntityHandlerTests { val jsonLdFile = ClassPathResource("/ngsild/aquac/fragments/DeadFishes_updateEntityAttribute.json") val entityId = "urn:ngsi-ld:DeadFishes:019BN".toUri() val updateResult = UpdateResult( - updated = arrayListOf( - UpdatedDetails( - fishNumberAttribute, - null, - UpdateOperationResult.REPLACED - ) - ), + updated = listOf(fishNumberAttribute), notUpdated = emptyList() ) @@ -1969,10 +1923,8 @@ class EntityHandlerTests { coEvery { entityService.updateAttributes(any(), any(), any()) } returns UpdateResult( - updated = arrayListOf( - UpdatedDetails(fishNumberAttribute, null, UpdateOperationResult.REPLACED) - ), - notUpdated = arrayListOf(notUpdatedAttribute) + updated = listOf(fishNumberAttribute), + notUpdated = listOf(notUpdatedAttribute) ).right() webClient.patch() @@ -2354,9 +2306,7 @@ class EntityHandlerTests { coEvery { entityService.replaceAttribute(any(), any(), any()) } returns UpdateResult( - updated = arrayListOf( - UpdatedDetails(INCOMING_PROPERTY, null, UpdateOperationResult.REPLACED) - ), + updated = listOf(INCOMING_PROPERTY), notUpdated = emptyList() ).right() diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/scope/ScopeServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/scope/ScopeServiceTests.kt index dca484c85..24bc39f48 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/scope/ScopeServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/scope/ScopeServiceTests.kt @@ -23,7 +23,6 @@ import com.egm.stellio.shared.util.shouldSucceed import com.egm.stellio.shared.util.shouldSucceedAndResult import com.egm.stellio.shared.util.shouldSucceedWith import com.egm.stellio.shared.util.toUri -import kotlinx.coroutines.runBlocking import kotlinx.coroutines.test.runTest import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.AfterEach @@ -45,7 +44,7 @@ import java.util.stream.Stream @SpringBootTest @ActiveProfiles("test") -class ScopeServiceTests : WithTimescaleContainer, WithKafkaContainer { +class ScopeServiceTests : WithTimescaleContainer, WithKafkaContainer() { @Autowired private lateinit var scopeService: ScopeService @@ -64,10 +63,6 @@ class ScopeServiceTests : WithTimescaleContainer, WithKafkaContainer { @AfterEach fun clearEntityPayloadTable() { r2dbcEntityTemplate.delete().from("entity_payload").all().block() - - runBlocking { - scopeService.delete(beehiveTestCId) - } } @Suppress("unused") diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/support/WithKafkaContainer.kt b/search-service/src/test/kotlin/com/egm/stellio/search/support/WithKafkaContainer.kt index f8f9fe1f2..727ef5444 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/support/WithKafkaContainer.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/support/WithKafkaContainer.kt @@ -1,29 +1,21 @@ package com.egm.stellio.search.support -import org.springframework.test.context.DynamicPropertyRegistry -import org.springframework.test.context.DynamicPropertySource +import org.springframework.boot.testcontainers.service.connection.ServiceConnection +import org.testcontainers.junit.jupiter.Container +import org.testcontainers.junit.jupiter.Testcontainers import org.testcontainers.kafka.ConfluentKafkaContainer -import org.testcontainers.utility.DockerImageName -interface WithKafkaContainer { +@Testcontainers +@Suppress("UtilityClassWithPublicConstructor") +open class WithKafkaContainer { companion object { - private val kafkaImage: DockerImageName = - DockerImageName.parse("confluentinc/cp-kafka:7.6.0") - - private val kafkaContainer = ConfluentKafkaContainer(kafkaImage).apply { - withReuse(true) - } - + @Container + @ServiceConnection @JvmStatic - @DynamicPropertySource - fun properties(registry: DynamicPropertyRegistry) { - registry.add("spring.kafka.bootstrap-servers") { kafkaContainer.bootstrapServers } - } - - init { - kafkaContainer.start() + val kafkaContainer = ConfluentKafkaContainer("confluentinc/cp-kafka:7.6.0").apply { + withReuse(true) } } } diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/AggregatedTemporalQueryServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/AggregatedTemporalQueryServiceTests.kt index 775715ee5..3efe15704 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/AggregatedTemporalQueryServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/AggregatedTemporalQueryServiceTests.kt @@ -45,7 +45,7 @@ import java.util.UUID @SpringBootTest @ActiveProfiles("test") -class AggregatedTemporalQueryServiceTests : WithTimescaleContainer, WithKafkaContainer { +class AggregatedTemporalQueryServiceTests : WithTimescaleContainer, WithKafkaContainer() { @Autowired private lateinit var attributeInstanceService: AttributeInstanceService diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/AttributeInstanceServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/AttributeInstanceServiceTests.kt index a05fc17f5..37f9eef69 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/AttributeInstanceServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/AttributeInstanceServiceTests.kt @@ -39,10 +39,12 @@ import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_VALUE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DATE_TIME_TYPE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DEFAULT_VOCAB +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DELETED_AT_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_INSTANCE_ID_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_JSONPROPERTY_VALUE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_LANGUAGEPROPERTY_VALUE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_MODIFIED_AT_PROPERTY +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_NULL import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_OBSERVED_AT_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_PROPERTY_VALUE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_VOCABPROPERTY_VALUE @@ -77,14 +79,12 @@ import org.springframework.boot.test.context.SpringBootTest import org.springframework.data.r2dbc.core.R2dbcEntityTemplate import org.springframework.r2dbc.core.DatabaseClient import org.springframework.test.context.ActiveProfiles -import java.time.Instant -import java.time.ZoneOffset import java.time.ZonedDateTime import java.util.UUID @SpringBootTest @ActiveProfiles("test") -class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer { +class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer() { @Autowired private lateinit var attributeInstanceService: AttributeInstanceService @@ -101,7 +101,7 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer @Autowired private lateinit var r2dbcEntityTemplate: R2dbcEntityTemplate - private val now = Instant.now().atZone(ZoneOffset.UTC) + private val now = ngsiLdDateTime() private lateinit var incomingAttribute: Attribute private lateinit var outgoingAttribute: Attribute private lateinit var jsonAttribute: Attribute @@ -331,7 +331,7 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer entityAttributeService.create(attribute2) (1..10).forEach { _ -> - val observedAt = Instant.now().atZone(ZoneOffset.UTC) + val observedAt = ngsiLdDateTime() val attributeMetadata = AttributeMetadata( measuredValue = null, value = "some value", @@ -613,7 +613,7 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer } @Test - fun `it should create an attribute instance if it has a non null value`() = runTest { + fun `it should create an observed attribute instance if it has a non null value`() = runTest { val attributeInstanceService = spyk( AttributeInstanceService(databaseClient, searchProperties), recordPrivateCalls = true @@ -641,7 +641,7 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer ) ) - attributeInstanceService.addAttributeInstance( + attributeInstanceService.addObservedAttributeInstance( incomingAttribute.id, attributeMetadata, attributeValues @@ -675,7 +675,7 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer } @Test - fun `it should create an attribute instance with boolean value`() = runTest { + fun `it should create an observed attribute instance with boolean value`() = runTest { val attributeInstanceService = spyk( AttributeInstanceService(databaseClient, searchProperties), recordPrivateCalls = true @@ -703,7 +703,7 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer ) ) - attributeInstanceService.addAttributeInstance( + attributeInstanceService.addObservedAttributeInstance( incomingAttribute.id, attributeMetadata, attributeValues @@ -736,6 +736,61 @@ class AttributeInstanceServiceTests : WithTimescaleContainer, WithKafkaContainer } } + @Test + fun `it should create a deleted attribute instance`() = runTest { + val attributeInstanceService = spyk( + AttributeInstanceService(databaseClient, searchProperties), + recordPrivateCalls = true + ) + val deletedAt = ZonedDateTime.parse("2025-01-02T11:20:30.000001Z") + val attributeValues = mapOf( + NGSILD_DELETED_AT_PROPERTY to listOf( + mapOf( + JSONLD_VALUE to deletedAt, + JSONLD_TYPE to NGSILD_DATE_TIME_TYPE + ) + ), + NGSILD_PROPERTY_VALUE to listOf( + mapOf( + JSONLD_VALUE to NGSILD_NULL + ) + ) + ) + + attributeInstanceService.addDeletedAttributeInstance( + incomingAttribute.id, + NGSILD_NULL, + deletedAt, + attributeValues + ) + + verify { + attributeInstanceService["create"]( + match { + it.time.toString() == "2025-01-02T11:20:30.000001Z" && + it.value == "urn:ngsi-ld:null" && + it.measuredValue == null && + it.payload.asString().matchContent( + """ + { + "https://uri.etsi.org/ngsi-ld/deletedAt":[{ + "@value":"2025-01-02T11:20:30.000001Z", + "@type":"https://uri.etsi.org/ngsi-ld/DateTime" + }], + "https://uri.etsi.org/ngsi-ld/hasValue":[{ + "@value":"urn:ngsi-ld:null" + }], + "https://uri.etsi.org/ngsi-ld/instanceId":[{ + "@id":"${it.instanceId}" + }] + } + """.trimIndent() + ) + } + ) + } + } + @Test fun `it should modify attribute instance for a property`() = runTest { val attributeInstance = gimmeNumericPropertyAttributeInstance(incomingAttribute.id) diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/TemporalQueryServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/TemporalQueryServiceTests.kt index e04ae3310..2a3cc15d9 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/TemporalQueryServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/TemporalQueryServiceTests.kt @@ -83,7 +83,7 @@ class TemporalQueryServiceTests { @Test fun `it should return an API exception if the entity does not exist`() = runTest { coEvery { - entityQueryService.checkEntityExistence(any()) + entityQueryService.retrieve(any(), any()) } returns ResourceNotFoundException(entityNotFoundMessage(entityUri.toString())).left() temporalQueryService.queryTemporalEntity( @@ -116,10 +116,9 @@ class TemporalQueryServiceTests { ) } - coEvery { entityQueryService.checkEntityExistence(any()) } returns Unit.right() - coEvery { authorizationService.userCanReadEntity(any(), any()) } returns Unit.right() - coEvery { entityAttributeService.getForEntity(any(), any(), any()) } returns attributes coEvery { entityQueryService.retrieve(any()) } returns gimmeEntityPayload().right() + coEvery { authorizationService.userCanReadEntity(any(), any()) } returns Unit.right() + coEvery { entityAttributeService.getForEntity(any(), any(), any(), any()) } returns attributes coEvery { scopeService.retrieveHistory(any(), any()) } returns emptyList().right() coEvery { attributeInstanceService.search(any(), any>()) @@ -147,9 +146,9 @@ class TemporalQueryServiceTests { ) coVerify { - entityQueryService.checkEntityExistence(entityUri) + entityQueryService.retrieve(entityUri) authorizationService.userCanReadEntity(entityUri, None) - entityAttributeService.getForEntity(entityUri, emptySet(), emptySet()) + entityAttributeService.getForEntity(entityUri, emptySet(), emptySet(), false) attributeInstanceService.search( match { temporalEntitiesQuery -> temporalEntitiesQuery.temporalQuery.timerel == TemporalQuery.Timerel.AFTER && @@ -246,11 +245,11 @@ class TemporalQueryServiceTests { ) coEvery { authorizationService.computeAccessRightFilter(any()) } returns { null } - coEvery { entityQueryService.queryEntities(any(), any<() -> String?>()) } returns listOf(entityUri) + coEvery { entityQueryService.queryEntities(any(), any(), any<() -> String?>()) } returns listOf(entityUri) coEvery { entityAttributeService.getForEntities(any(), any()) } returns listOf(attribute) - coEvery { entityQueryService.queryEntitiesCount(any(), any()) } returns 1.right() + coEvery { entityQueryService.queryEntitiesCount(any(), any(), any()) } returns 1.right() coEvery { scopeService.retrieveHistory(any(), any()) } returns emptyList().right() coEvery { entityQueryService.retrieve(any()) } returns gimmeEntityPayload().right() coEvery { @@ -305,6 +304,7 @@ class TemporalQueryServiceTests { paginationQuery = PaginationQuery(limit = 2, offset = 2), contexts = APIC_COMPOUND_CONTEXTS ), + false, any() ) scopeService.retrieveHistory(listOf(entityUri), any()) @@ -322,7 +322,7 @@ class TemporalQueryServiceTests { ) coEvery { authorizationService.computeAccessRightFilter(any()) } returns { null } - coEvery { entityQueryService.queryEntities(any(), any<() -> String?>()) } returns listOf(entityUri) + coEvery { entityQueryService.queryEntities(any(), any(), any<() -> String?>()) } returns listOf(entityUri) coEvery { entityAttributeService.getForEntities(any(), any()) } returns listOf(attribute) @@ -331,7 +331,7 @@ class TemporalQueryServiceTests { attributeInstanceService.search(any(), any>()) } returns emptyList().right() coEvery { entityQueryService.retrieve(any()) } returns gimmeEntityPayload().right() - coEvery { entityQueryService.queryEntitiesCount(any(), any()) } returns 1.right() + coEvery { entityQueryService.queryEntitiesCount(any(), any(), any()) } returns 1.right() temporalQueryService.queryTemporalEntities( TemporalEntitiesQueryFromGet( diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/TemporalServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/TemporalServiceTests.kt new file mode 100644 index 000000000..9e04afa27 --- /dev/null +++ b/search-service/src/test/kotlin/com/egm/stellio/search/temporal/service/TemporalServiceTests.kt @@ -0,0 +1,127 @@ +package com.egm.stellio.search.temporal.service + +import arrow.core.left +import arrow.core.right +import com.egm.stellio.search.authorization.service.AuthorizationService +import com.egm.stellio.search.entity.service.EntityQueryService +import com.egm.stellio.search.entity.service.EntityService +import com.egm.stellio.shared.model.ResourceNotFoundException +import com.egm.stellio.shared.util.INCOMING_PROPERTY +import com.egm.stellio.shared.util.loadAndExpandSampleData +import com.egm.stellio.shared.util.shouldSucceed +import com.egm.stellio.shared.util.toUri +import com.ninjasquad.springmockk.MockkBean +import io.mockk.coEvery +import io.mockk.coVerify +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.test.context.ActiveProfiles + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, classes = [TemporalService::class]) +@ActiveProfiles("test") +@ExperimentalCoroutinesApi +class TemporalServiceTests { + + @Autowired + private lateinit var temporalService: TemporalService + + @MockkBean + private lateinit var entityQueryService: EntityQueryService + + @MockkBean + private lateinit var entityService: EntityService + + @MockkBean + private lateinit var attributeInstanceService: AttributeInstanceService + + @MockkBean + private lateinit var authorizationService: AuthorizationService + + private val entityUri = "urn:ngsi-ld:BeeHive:TESTC".toUri() + private val sub = "0123456789-1234-5678-987654321" + + private fun mockkAuthorizationForCreation() { + coEvery { authorizationService.userCanCreateEntities(any()) } returns Unit.right() + coEvery { authorizationService.userCanUpdateEntity(any(), any()) } returns Unit.right() + coEvery { authorizationService.createOwnerRight(any(), any()) } returns Unit.right() + } + + @Test + fun `it should ask to create a temporal entity if it does not exist yet`() = runTest { + mockkAuthorizationForCreation() + coEvery { + entityQueryService.isMarkedAsDeleted(entityUri) + } returns ResourceNotFoundException("Entity does not exist").left() + coEvery { entityService.createEntity(any(), any(), any()) } returns Unit.right() + coEvery { entityService.upsertAttributes(any(), any(), any()) } returns Unit.right() + + val expandedEntity = loadAndExpandSampleData("temporal/beehive_create_temporal_entity.jsonld") + + temporalService.createOrUpdateTemporalEntity(entityUri, expandedEntity, sub).shouldSucceed() + } + + @Test + fun `it should ask to create a temporal entity if it already exists but is deleted`() = runTest { + mockkAuthorizationForCreation() + coEvery { entityQueryService.isMarkedAsDeleted(entityUri) } returns false.right() + coEvery { entityService.createEntity(any(), any(), any()) } returns Unit.right() + coEvery { entityService.upsertAttributes(any(), any(), any()) } returns Unit.right() + + val expandedEntity = loadAndExpandSampleData("temporal/beehive_create_temporal_entity.jsonld") + + temporalService.createOrUpdateTemporalEntity(entityUri, expandedEntity, sub).shouldSucceed() + } + + @Test + fun `it should ask to upsert a temporal entity if it already exists but is not deleted`() = runTest { + mockkAuthorizationForCreation() + coEvery { entityQueryService.isMarkedAsDeleted(entityUri) } returns false.right() + coEvery { entityService.upsertAttributes(any(), any(), any()) } returns Unit.right() + + val expandedEntity = loadAndExpandSampleData("temporal/beehive_create_temporal_entity.jsonld") + + temporalService.createOrUpdateTemporalEntity(entityUri, expandedEntity, sub).shouldSucceed() + } + + @Test + fun `it should ask to permanently delete a temporal entity`() = runTest { + coEvery { entityService.permanentlyDeleteEntity(any(), any()) } returns Unit.right() + + temporalService.deleteEntity(entityUri, sub).shouldSucceed() + + coVerify(exactly = 1) { + entityService.permanentlyDeleteEntity(entityUri, sub) + } + } + + @Test + fun `it should ask to permanently delete a temporal attribute`() = runTest { + coEvery { entityService.permanentlyDeleteAttribute(any(), any(), any(), any(), any()) } returns Unit.right() + + temporalService.deleteAttribute(entityUri, INCOMING_PROPERTY, null).shouldSucceed() + + coVerify(exactly = 1) { + entityService.permanentlyDeleteAttribute(entityUri, INCOMING_PROPERTY, null) + } + } + + @Test + fun `it should ask to permanently delete a temporal attribute instance`() = runTest { + val instanceId = "urn:ngsi-ld:Instance:01".toUri() + + coEvery { entityQueryService.checkEntityExistence(entityUri) } returns Unit.right() + coEvery { authorizationService.userCanUpdateEntity(entityUri, any()) } returns Unit.right() + coEvery { + attributeInstanceService.deleteInstance(entityUri, INCOMING_PROPERTY, instanceId) + } returns Unit.right() + + temporalService.deleteAttributeInstance(entityUri, INCOMING_PROPERTY, instanceId).shouldSucceed() + + coVerify(exactly = 1) { + attributeInstanceService.deleteInstance(entityUri, INCOMING_PROPERTY, instanceId) + } + } +} diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/temporal/web/TemporalEntityHandlerTestCommon.kt b/search-service/src/test/kotlin/com/egm/stellio/search/temporal/web/TemporalEntityHandlerTestCommon.kt index 0102fd19a..f2462beff 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/temporal/web/TemporalEntityHandlerTestCommon.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/temporal/web/TemporalEntityHandlerTestCommon.kt @@ -1,7 +1,6 @@ package com.egm.stellio.search.temporal.web import com.egm.stellio.search.common.config.SearchProperties -import com.egm.stellio.search.entity.service.EntityService import com.egm.stellio.search.temporal.service.TemporalQueryService import com.egm.stellio.search.temporal.service.TemporalService import com.egm.stellio.shared.config.ApplicationProperties @@ -30,9 +29,6 @@ open class TemporalEntityHandlerTestCommon { @MockkBean(relaxed = true) protected lateinit var temporalQueryService: TemporalQueryService - @MockkBean - protected lateinit var entityService: EntityService - @BeforeAll fun configureWebClientDefaults() { webClient = webClient.mutate() diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/temporal/web/TemporalEntityHandlerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/temporal/web/TemporalEntityHandlerTests.kt index 24b3a4126..73d2f7541 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/temporal/web/TemporalEntityHandlerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/temporal/web/TemporalEntityHandlerTests.kt @@ -972,7 +972,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { @Test fun `delete temporal entity should return a 204 if an entity has been successfully deleted`() { - coEvery { entityService.deleteEntity(any(), any()) } returns Unit.right() + coEvery { temporalService.deleteEntity(any(), any()) } returns Unit.right() webClient.delete() .uri("/ngsi-ld/v1/temporal/entities/$entityUri") @@ -981,14 +981,14 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { .expectBody().isEmpty coVerify { - entityService.deleteEntity(eq(entityUri), eq(sub.value)) + temporalService.deleteEntity(eq(entityUri), eq(sub.value)) } } @Test fun `delete temporal entity should return a 404 if entity to be deleted has not been found`() { coEvery { - entityService.deleteEntity(entityUri, sub.getOrNull()) + temporalService.deleteEntity(entityUri, sub.getOrNull()) } returns ResourceNotFoundException(entityNotFoundMessage(entityUri.toString())).left() webClient.delete() @@ -1034,7 +1034,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { @Test fun `delete temporal entity should return a 500 if entity could not be deleted`() { coEvery { - entityService.deleteEntity(any(), any()) + temporalService.deleteEntity(any(), any()) } throws RuntimeException("Unexpected server error") webClient.delete() @@ -1055,7 +1055,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { @Test fun `delete temporal entity should return a 403 is user is not authorized to delete an entity`() { coEvery { - entityService.deleteEntity(any(), any()) + temporalService.deleteEntity(any(), any()) } returns AccessDeniedException("User forbidden admin access to entity $entityUri").left() webClient.delete() @@ -1077,7 +1077,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { @Test fun `delete attribute temporal should return a 204 if the attribute has been successfully deleted`() { coEvery { - entityService.deleteAttribute(any(), any(), any(), any(), any()) + temporalService.deleteAttribute(any(), any(), any(), any(), any()) } returns Unit.right() webClient.method(HttpMethod.DELETE) @@ -1089,7 +1089,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { .expectBody().isEmpty coVerify { - entityService.deleteAttribute( + temporalService.deleteAttribute( eq(entityUri), eq(TEMPERATURE_PROPERTY), null, @@ -1102,7 +1102,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { @Test fun `delete attribute temporal should delete all instances if deleteAll flag is true`() { coEvery { - entityService.deleteAttribute(any(), any(), any(), any(), any()) + temporalService.deleteAttribute(any(), any(), any(), any(), any()) } returns Unit.right() webClient.method(HttpMethod.DELETE) @@ -1114,7 +1114,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { .expectBody().isEmpty coVerify { - entityService.deleteAttribute( + temporalService.deleteAttribute( eq(entityUri), eq(TEMPERATURE_PROPERTY), null, @@ -1128,7 +1128,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { fun `delete attribute temporal should delete instance with the provided datasetId`() { val datasetId = "urn:ngsi-ld:Dataset:temperature:1" coEvery { - entityService.deleteAttribute(any(), any(), any(), any(), any()) + temporalService.deleteAttribute(any(), any(), any(), any(), any()) } returns Unit.right() webClient.method(HttpMethod.DELETE) @@ -1140,7 +1140,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { .expectBody().isEmpty coVerify { - entityService.deleteAttribute( + temporalService.deleteAttribute( eq(entityUri), eq(TEMPERATURE_PROPERTY), eq(datasetId.toUri()), @@ -1153,7 +1153,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { @Test fun `delete attribute temporal should return a 404 if the entity is not found`() { coEvery { - entityService.deleteAttribute(any(), any(), any(), any(), any()) + temporalService.deleteAttribute(any(), any(), any(), any(), any()) } returns ResourceNotFoundException(entityNotFoundMessage(entityUri.toString())).left() webClient.method(HttpMethod.DELETE) @@ -1176,7 +1176,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { @Test fun `delete attribute temporal should return a 404 if the attribute is not found`() { coEvery { - entityService.deleteAttribute(any(), any(), any(), any(), any()) + temporalService.deleteAttribute(any(), any(), any(), any(), any()) } returns ResourceNotFoundException("Attribute Not Found").left() webClient.method(HttpMethod.DELETE) @@ -1199,7 +1199,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { @Test fun `delete attribute temporal should return a 400 if the request is not correct`() { coEvery { - entityService.deleteAttribute(any(), any(), any(), any(), any()) + temporalService.deleteAttribute(any(), any(), any(), any(), any()) } returns BadRequestDataException("Something is wrong with the request").left() webClient.method(HttpMethod.DELETE) @@ -1273,7 +1273,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() { @Test fun `delete attribute temporal should return a 403 if user is not allowed to update entity`() { coEvery { - entityService.deleteAttribute(any(), any(), any(), any(), any()) + temporalService.deleteAttribute(any(), any(), any(), any(), any()) } returns AccessDeniedException("User forbidden write access to entity $entityUri").left() webClient.method(HttpMethod.DELETE) diff --git a/search-service/src/test/resources/ngsild/fragments/beehive_mergeAttribute_null.json b/search-service/src/test/resources/ngsild/fragments/beehive_mergeAttribute_null.json new file mode 100644 index 000000000..0f3603a67 --- /dev/null +++ b/search-service/src/test/resources/ngsild/fragments/beehive_mergeAttribute_null.json @@ -0,0 +1,6 @@ +{ + "incoming": { + "type": "Property", + "value": "urn:ngsi-ld:null" + } +} diff --git a/search-service/src/test/resources/ngsild/fragments/beehive_mergeAttribute_null_fragment.json b/search-service/src/test/resources/ngsild/fragments/beehive_mergeAttribute_null_fragment.json new file mode 100644 index 000000000..acf095c32 --- /dev/null +++ b/search-service/src/test/resources/ngsild/fragments/beehive_mergeAttribute_null_fragment.json @@ -0,0 +1,4 @@ +{ + "type": "Property", + "value": "urn:ngsi-ld:null" +} diff --git a/shared/config/detekt/baseline.xml b/shared/config/detekt/baseline.xml index c359f0fb8..849a3ca3e 100644 --- a/shared/config/detekt/baseline.xml +++ b/shared/config/detekt/baseline.xml @@ -7,7 +7,6 @@ LongMethod:QueryUtils.kt$private fun transformQQueryToSqlJsonPath( mainAttributePath: List<ExpandedTerm>, trailingAttributePath: List<ExpandedTerm>, operator: String, value: String ) LongParameterList:ApiResponses.kt$( body: String, count: Int, resourceUrl: String, paginationQuery: PaginationQuery, requestParams: MultiValueMap<String, String>, mediaType: MediaType, contexts: List<String> ) LongParameterList:ApiResponses.kt$( entities: Any, count: Int, resourceUrl: String, paginationQuery: PaginationQuery, requestParams: MultiValueMap<String, String>, mediaType: MediaType, contexts: List<String> ) - SpreadOperator:EntityEvent.kt$EntityEvent$( *[ JsonSubTypes.Type(value = EntityCreateEvent::class), JsonSubTypes.Type(value = EntityReplaceEvent::class), JsonSubTypes.Type(value = EntityDeleteEvent::class), JsonSubTypes.Type(value = AttributeAppendEvent::class), JsonSubTypes.Type(value = AttributeReplaceEvent::class), JsonSubTypes.Type(value = AttributeUpdateEvent::class), JsonSubTypes.Type(value = AttributeDeleteEvent::class), JsonSubTypes.Type(value = AttributeDeleteAllInstancesEvent::class) ] ) TooManyFunctions:JsonLdUtils.kt$JsonLdUtils diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/model/CompactedEntity.kt b/shared/src/main/kotlin/com/egm/stellio/shared/model/CompactedEntity.kt index 6eb784a24..cf9e8d562 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/model/CompactedEntity.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/model/CompactedEntity.kt @@ -6,6 +6,7 @@ import com.egm.stellio.shared.model.AttributeCompactedType.LANGUAGEPROPERTY import com.egm.stellio.shared.model.AttributeCompactedType.PROPERTY import com.egm.stellio.shared.model.AttributeCompactedType.RELATIONSHIP import com.egm.stellio.shared.model.AttributeCompactedType.VOCABPROPERTY +import com.egm.stellio.shared.model.AttributeCompactedType.entries import com.egm.stellio.shared.queryparameter.QueryParameter import com.egm.stellio.shared.util.FEATURES_PROPERTY_TERM import com.egm.stellio.shared.util.FEATURE_COLLECTION_TYPE @@ -23,6 +24,7 @@ import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_VOCAB_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_CREATED_AT_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DATASET_ID_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DATASET_TERM +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DELETED_AT_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_ENTITY_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_GEOPROPERTY_TERM import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_JSONPROPERTY_TERM @@ -39,8 +41,7 @@ import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_VOCABPROPERTY_TERM import com.egm.stellio.shared.util.PROPERTIES_PROPERTY_TERM import com.egm.stellio.shared.util.toUri import java.net.URI -import java.util.Locale -import kotlin.collections.Map +import java.util.* typealias CompactedEntity = Map typealias CompactedAttributeInstance = Map @@ -220,7 +221,8 @@ fun CompactedEntity.withoutSysAttrs(sysAttrToKeep: String?): Map { } return this.filter { - !sysAttrsToRemove.contains(it.key) + // deletedAt has to be kept at entity level (but not in attributes), see 5.8.6 + !sysAttrsToRemove.minus(NGSILD_DELETED_AT_TERM).contains(it.key) }.mapValues { when (it.value) { is Map<*, *> -> removeSysAttrsFromAttrInstance(it.value as Map<*, *>) diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/model/EntityEvent.kt b/shared/src/main/kotlin/com/egm/stellio/shared/model/EntityEvent.kt index 1de2e0332..9005ce8ae 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/model/EntityEvent.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/model/EntityEvent.kt @@ -9,15 +9,14 @@ import java.net.URI @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.EXISTING_PROPERTY, property = "operationType") @JsonSubTypes( - *[ + value = [ JsonSubTypes.Type(value = EntityCreateEvent::class), JsonSubTypes.Type(value = EntityReplaceEvent::class), JsonSubTypes.Type(value = EntityDeleteEvent::class), JsonSubTypes.Type(value = AttributeAppendEvent::class), JsonSubTypes.Type(value = AttributeReplaceEvent::class), JsonSubTypes.Type(value = AttributeUpdateEvent::class), - JsonSubTypes.Type(value = AttributeDeleteEvent::class), - JsonSubTypes.Type(value = AttributeDeleteAllInstancesEvent::class) + JsonSubTypes.Type(value = AttributeDeleteEvent::class) ] ) sealed class EntityEvent( @@ -74,10 +73,11 @@ data class EntityDeleteEvent( override val entityId: URI, override val entityTypes: List, // null only when in the case of an IAM event (previous state is not known) - val deletedEntity: String?, + val previousEntity: String?, + val updatedEntity: String, override val contexts: List ) : EntityEvent(EventsType.ENTITY_DELETE, sub, tenantName, entityId, entityTypes, contexts) { - override fun getEntity() = this.deletedEntity + override fun getEntity() = this.previousEntity } @JsonTypeName("ATTRIBUTE_APPEND") @@ -88,7 +88,6 @@ data class AttributeAppendEvent( override val entityTypes: List, val attributeName: ExpandedTerm, val datasetId: URI?, - val overwrite: Boolean = true, val operationPayload: String, val updatedEntity: String, override val contexts: List @@ -144,20 +143,6 @@ data class AttributeDeleteEvent( override fun getAttribute() = this.attributeName } -@JsonTypeName("ATTRIBUTE_DELETE_ALL_INSTANCES") -data class AttributeDeleteAllInstancesEvent( - override val sub: String?, - override val tenantName: String = DEFAULT_TENANT_NAME, - override val entityId: URI, - override val entityTypes: List, - val attributeName: ExpandedTerm, - val updatedEntity: String, - override val contexts: List -) : EntityEvent(EventsType.ATTRIBUTE_DELETE_ALL_INSTANCES, sub, tenantName, entityId, entityTypes, contexts) { - override fun getEntity() = this.updatedEntity - override fun getAttribute() = this.attributeName -} - enum class EventsType { ENTITY_CREATE, ENTITY_REPLACE, @@ -165,8 +150,7 @@ enum class EventsType { ATTRIBUTE_APPEND, ATTRIBUTE_REPLACE, ATTRIBUTE_UPDATE, - ATTRIBUTE_DELETE, - ATTRIBUTE_DELETE_ALL_INSTANCES + ATTRIBUTE_DELETE } fun unhandledOperationType(operationType: EventsType): String = "Entity event $operationType not handled." diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/model/ExpandedMembers.kt b/shared/src/main/kotlin/com/egm/stellio/shared/model/ExpandedMembers.kt index b4ed57866..08242e58f 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/model/ExpandedMembers.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/model/ExpandedMembers.kt @@ -11,6 +11,7 @@ import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_CREATED_AT_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DATASET_ID_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DATE_TIME_TYPE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DATE_TYPE +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DELETED_AT_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_MODIFIED_AT_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_PROPERTY_VALUE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_RELATIONSHIP_OBJECT @@ -91,8 +92,9 @@ fun ExpandedAttributeInstances.getSingleEntry(): ExpandedAttributeInstance { fun ExpandedAttributeInstance.addSysAttrs( withSysAttrs: Boolean, createdAt: ZonedDateTime, - modifiedAt: ZonedDateTime? -): Map = + modifiedAt: ZonedDateTime? = null, + deletedAt: ZonedDateTime? = null +): ExpandedAttributeInstance = if (withSysAttrs) this.plus(NGSILD_CREATED_AT_PROPERTY to buildNonReifiedTemporalValue(createdAt)) .let { @@ -100,6 +102,11 @@ fun ExpandedAttributeInstance.addSysAttrs( it.plus(NGSILD_MODIFIED_AT_PROPERTY to buildNonReifiedTemporalValue(modifiedAt)) else it } + .let { + if (deletedAt != null) + it.plus(NGSILD_DELETED_AT_PROPERTY to buildNonReifiedTemporalValue(deletedAt)) + else it + } else this /** @@ -197,6 +204,9 @@ fun ExpandedAttributeInstance.getRelationshipObject(name: String): Either)?.get(JSONLD_ID)?.toUri() +fun ExpandedAttributeInstance.getRelationshipId(): URI? = + (this[NGSILD_RELATIONSHIP_OBJECT]?.get(0) as? Map)?.get(JSONLD_ID)?.toUri() + fun ExpandedAttributeInstance.getScopes(): List? = when (val rawScopes = this.getMemberValue(NGSILD_SCOPE_PROPERTY)) { is String -> listOf(rawScopes) diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/queryparameter/QueryParameter.kt b/shared/src/main/kotlin/com/egm/stellio/shared/queryparameter/QueryParameter.kt index 00d91392b..3762001e6 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/queryparameter/QueryParameter.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/queryparameter/QueryParameter.kt @@ -35,6 +35,9 @@ enum class QueryParameter( LASTN("lastN"), TIMEPROPERTY("timeproperty"), + // authz + INCLUDE_DELETED("includeDeleted"), + // pagination COUNT("count"), OFFSET("offset"), diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt index e05b2391d..8fe916f65 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiResponses.kt @@ -56,7 +56,6 @@ const val INCONSISTENT_VALUES_IN_AGGREGATION_MESSAGE = const val ENTITIY_CREATION_FORBIDDEN_MESSAGE = "User forbidden to create entity" const val ENTITIY_READ_FORBIDDEN_MESSAGE = "User forbidden to read entity" const val ENTITY_UPDATE_FORBIDDEN_MESSAGE = "User forbidden to modify entity" -const val ENTITY_DELETE_FORBIDDEN_MESSAGE = "User forbidden to delete entity" const val ENTITY_ADMIN_FORBIDDEN_MESSAGE = "User forbidden to admin entity" const val ENTITY_REMOVE_OWNERSHIP_FORBIDDEN_MESSAGE = "User forbidden to remove ownership of entity" const val ENTITY_ALREADY_EXISTS_MESSAGE = "Entity already exists" diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/AuthUtils.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/AuthUtils.kt index 0d10c7f76..9bbf3ed68 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/AuthUtils.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/AuthUtils.kt @@ -67,6 +67,8 @@ object AuthContextModel { const val AUTH_PROP_SAP = AUTHORIZATION_ONTOLOGY + AUTH_TERM_SAP const val AUTH_TERM_RIGHT = "right" const val AUTH_PROP_RIGHT: ExpandedTerm = AUTHORIZATION_ONTOLOGY + AUTH_TERM_RIGHT + const val AUTH_TERM_IS_DELETED = "isDeleted" + const val AUTH_PROP_IS_DELETED: ExpandedTerm = AUTHORIZATION_ONTOLOGY + AUTH_TERM_IS_DELETED const val AUTH_TERM_IS_MEMBER_OF = "isMemberOf" const val AUTH_REL_IS_MEMBER_OF: ExpandedTerm = AUTHORIZATION_ONTOLOGY + AUTH_TERM_IS_MEMBER_OF diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/DataRepresentationUtils.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/DataRepresentationUtils.kt index 0ec2645aa..5aa04074e 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/DataRepresentationUtils.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/DataRepresentationUtils.kt @@ -40,9 +40,9 @@ fun String.checkNameIsNgsiLdSupported(): Either = * Returns whether the given string is a supported name as defined in 4.6.2 */ private fun String.isNgsiLdSupportedName(): Boolean = - this.all { char -> char.isLetterOrDigit() || listOf(':', '_').contains(char) } + this.all { char -> char.isLetterOrDigit() || listOf(':', '_', '@').contains(char) } -val scopeNameRegex: Regex = """^/?\p{L}+[\p{L}\w_]*(/\p{L}+[\p{L}\w_]*)*$""".toRegex() +val scopeNameRegex: Regex = """^/?\p{L}+[\p{L}\w]*(/\p{L}+[\p{L}\w]*)*$""".toRegex() fun Any.checkScopesNamesAreNgsiLdSupported(): Either { val rawScope = this diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt index c07320615..58681c694 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/JsonLdUtils.kt @@ -94,6 +94,8 @@ object JsonLdUtils { const val NGSILD_NONE_TERM = "@none" const val NGSILD_DATASET_TERM = "dataset" const val NGSILD_ENTITY_TERM = "entity" + const val NGSILD_NULL = "urn:ngsi-ld:null" + val JSONLD_EXPANDED_ENTITY_SPECIFIC_MEMBERS = setOf(JSONLD_TYPE, NGSILD_SCOPE_PROPERTY) // List of members that are part of a core entity base definition (i.e., without attributes) @@ -118,12 +120,14 @@ object JsonLdUtils { const val NGSILD_CREATED_AT_TERM = "createdAt" const val NGSILD_MODIFIED_AT_TERM = "modifiedAt" - val NGSILD_SYSATTRS_TERMS = setOf(NGSILD_CREATED_AT_TERM, NGSILD_MODIFIED_AT_TERM) + val NGSILD_SYSATTRS_TERMS = setOf(NGSILD_CREATED_AT_TERM, NGSILD_MODIFIED_AT_TERM, NGSILD_DELETED_AT_TERM) const val NGSILD_CREATED_AT_PROPERTY = "https://uri.etsi.org/ngsi-ld/$NGSILD_CREATED_AT_TERM" const val NGSILD_MODIFIED_AT_PROPERTY = "https://uri.etsi.org/ngsi-ld/$NGSILD_MODIFIED_AT_TERM" val NGSILD_SYSATTRS_PROPERTIES = setOf(NGSILD_CREATED_AT_PROPERTY, NGSILD_MODIFIED_AT_PROPERTY) const val NGSILD_OBSERVED_AT_TERM = "observedAt" const val NGSILD_OBSERVED_AT_PROPERTY = "https://uri.etsi.org/ngsi-ld/$NGSILD_OBSERVED_AT_TERM" + const val NGSILD_DELETED_AT_TERM = "deletedAt" + const val NGSILD_DELETED_AT_PROPERTY = "https://uri.etsi.org/ngsi-ld/$NGSILD_DELETED_AT_TERM" const val NGSILD_UNIT_CODE_PROPERTY = "https://uri.etsi.org/ngsi-ld/unitCode" const val NGSILD_UNIT_CODE_TERM = "unitCode" const val NGSILD_LOCATION_TERM = "location" @@ -295,13 +299,13 @@ object JsonLdUtils { if (NGSILD_GEO_PROPERTIES_TERMS.contains(it.key)) { when (it.value) { is Map<*, *> -> { - val geoProperty = it.value as MutableMap - val wktGeometry = throwingGeoJsonToWkt(geoProperty[JSONLD_VALUE_TERM]!! as Map) + val geoProperty = it.value as Map + val wktGeometry = geoPropertyToWKTOrNull(geoProperty[JSONLD_VALUE_TERM]!!) geoProperty.plus(JSONLD_VALUE_TERM to wktGeometry) } is List<*> -> { (it.value as List>).map { geoProperty -> - val wktGeometry = throwingGeoJsonToWkt(geoProperty[JSONLD_VALUE_TERM] as Map) + val wktGeometry = geoPropertyToWKTOrNull(geoProperty[JSONLD_VALUE_TERM]!!) geoProperty.plus(JSONLD_VALUE_TERM to wktGeometry) } } @@ -310,6 +314,12 @@ object JsonLdUtils { } else it.value } + private fun geoPropertyToWKTOrNull(geoPropertyValue: Any): String = + if (geoPropertyValue is String && geoPropertyValue == NGSILD_NULL) + NGSILD_NULL + else + throwingGeoJsonToWkt(geoPropertyValue as Map) + private fun restoreGeoPropertyFromWKT(): (Map.Entry) -> Any = { if (NGSILD_GEO_PROPERTIES_TERMS.contains(it.key)) { when (it.value) { diff --git a/shared/src/test/kotlin/com/egm/stellio/shared/model/ExpandedMembersTests.kt b/shared/src/test/kotlin/com/egm/stellio/shared/model/ExpandedMembersTests.kt index 51e49eee4..d91d68a38 100644 --- a/shared/src/test/kotlin/com/egm/stellio/shared/model/ExpandedMembersTests.kt +++ b/shared/src/test/kotlin/com/egm/stellio/shared/model/ExpandedMembersTests.kt @@ -2,6 +2,7 @@ package com.egm.stellio.shared.model import com.egm.stellio.shared.util.JsonLdUtils import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_CREATED_AT_PROPERTY +import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DELETED_AT_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_MODIFIED_AT_PROPERTY import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_RELATIONSHIP_OBJECT import com.egm.stellio.shared.util.JsonLdUtils.buildExpandedPropertyValue @@ -30,6 +31,7 @@ class ExpandedMembersTests { assertThat(attrPayloadWithSysAttrs) .containsKey(NGSILD_CREATED_AT_PROPERTY) .doesNotContainKey(NGSILD_MODIFIED_AT_PROPERTY) + .doesNotContainKey(NGSILD_DELETED_AT_PROPERTY) } @Test @@ -41,6 +43,20 @@ class ExpandedMembersTests { assertThat(attrPayloadWithSysAttrs) .containsKey(NGSILD_CREATED_AT_PROPERTY) .containsKey(NGSILD_MODIFIED_AT_PROPERTY) + .doesNotContainKey(NGSILD_DELETED_AT_PROPERTY) + } + + @Test + fun `it should add createdAt, modifiedAt and deletedAt information into an attribute`() { + val attrPayload = mapOf("attribute" to buildExpandedPropertyValue(12.0)) + + val attrPayloadWithSysAttrs = + attrPayload.addSysAttrs(true, ngsiLdDateTime(), ngsiLdDateTime(), ngsiLdDateTime()) + + assertThat(attrPayloadWithSysAttrs) + .containsKey(NGSILD_CREATED_AT_PROPERTY) + .containsKey(NGSILD_MODIFIED_AT_PROPERTY) + .containsKey(NGSILD_DELETED_AT_PROPERTY) } @Test diff --git a/shared/src/test/kotlin/com/egm/stellio/shared/util/JsonUtilsTests.kt b/shared/src/test/kotlin/com/egm/stellio/shared/util/JsonUtilsTests.kt index ddad6db9b..5c3563fcb 100644 --- a/shared/src/test/kotlin/com/egm/stellio/shared/util/JsonUtilsTests.kt +++ b/shared/src/test/kotlin/com/egm/stellio/shared/util/JsonUtilsTests.kt @@ -1,6 +1,5 @@ package com.egm.stellio.shared.util -import com.egm.stellio.shared.model.AttributeDeleteAllInstancesEvent import com.egm.stellio.shared.model.AttributeDeleteEvent import com.egm.stellio.shared.model.AttributeReplaceEvent import com.egm.stellio.shared.model.AttributeUpdateEvent @@ -19,6 +18,7 @@ import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows +import java.time.ZonedDateTime class JsonUtilsTests { @@ -114,14 +114,6 @@ class JsonUtilsTests { Assertions.assertTrue(parsedEvent is AttributeDeleteEvent) } - @Test - fun `it should parse an event of type ATTRIBUTE_DELETE_ALL_INSTANCES`() { - val parsedEvent = deserializeAs( - loadSampleData("events/entity/attributeDeleteAllInstancesEvent.json") - ) - Assertions.assertTrue(parsedEvent is AttributeDeleteAllInstancesEvent) - } - @Test fun `it should serialize an event of type ENTITY_CREATE`() = runTest { val event = mapper.writeValueAsString( @@ -146,6 +138,13 @@ class JsonUtilsTests { entityId, listOf(BEEHIVE_TYPE), serializeObject(expandJsonLdFragment(entityPayload, APIC_COMPOUND_CONTEXTS)), + serializeObject( + loadAndExpandDeletedEntity( + entityId, + ZonedDateTime.parse("2024-12-23T17:01:02Z"), + APIC_COMPOUND_CONTEXTS + ).members + ), emptyList() ) ) diff --git a/shared/src/testFixtures/kotlin/com/egm/stellio/shared/util/TestUtils.kt b/shared/src/testFixtures/kotlin/com/egm/stellio/shared/util/TestUtils.kt index 167737ce4..69723ecd9 100644 --- a/shared/src/testFixtures/kotlin/com/egm/stellio/shared/util/TestUtils.kt +++ b/shared/src/testFixtures/kotlin/com/egm/stellio/shared/util/TestUtils.kt @@ -14,6 +14,7 @@ import com.egm.stellio.shared.util.JsonLdUtils.expandJsonLdEntity import com.egm.stellio.shared.util.JsonUtils.deserializeAsMap import org.springframework.core.io.ClassPathResource import java.net.URI +import java.time.ZonedDateTime fun loadSampleData(filename: String = "beehive.jsonld"): String { val sampleData = ClassPathResource("/ngsild/$filename") @@ -86,6 +87,21 @@ suspend fun loadAndExpandMinimalEntity( contexts ) +suspend fun loadAndExpandDeletedEntity( + entityId: URI, + deletedAt: ZonedDateTime? = ngsiLdDateTime(), + contexts: List = APIC_COMPOUND_CONTEXTS +): ExpandedEntity = + expandJsonLdEntity( + """ + { + "id": "$entityId", + "deletedAt": "$deletedAt" + } + """.trimIndent(), + contexts + ) + suspend fun String.sampleDataToNgsiLdEntity(): Either> { val expandedEntity = expandJsonLdEntity(this) return when (val ngsiLdEntity = expandedEntity.toNgsiLdEntity()) { diff --git a/shared/src/testFixtures/resources/jsonld-contexts/authorization.jsonld b/shared/src/testFixtures/resources/jsonld-contexts/authorization.jsonld index 28361e902..f086636cc 100644 --- a/shared/src/testFixtures/resources/jsonld-contexts/authorization.jsonld +++ b/shared/src/testFixtures/resources/jsonld-contexts/authorization.jsonld @@ -9,6 +9,7 @@ "clientId": "https://ontology.eglobalmark.com/authorization#clientId", "familyName": "https://ontology.eglobalmark.com/authorization#familyName", "givenName": "https://ontology.eglobalmark.com/authorization#givenName", + "isDeleted": "https://ontology.eglobalmark.com/authorization#isDeleted", "isMemberOf": "https://ontology.eglobalmark.com/authorization#isMemberOf", "isOwner": "https://ontology.eglobalmark.com/authorization#isOwner", "kind": "https://ontology.eglobalmark.com/authorization#kind", diff --git a/shared/src/testFixtures/resources/ngsild/events/authorization/RightAddOnEntity.json b/shared/src/testFixtures/resources/ngsild/events/authorization/RightAddOnEntity.json index 119a873df..bc05edeab 100644 --- a/shared/src/testFixtures/resources/ngsild/events/authorization/RightAddOnEntity.json +++ b/shared/src/testFixtures/resources/ngsild/events/authorization/RightAddOnEntity.json @@ -8,6 +8,5 @@ "contexts": [ "https://easy-global-market.github.io/ngsild-api-data-models/authorization/jsonld-contexts/authorization-compound.jsonld" ], - "overwrite": true, "operationType": "ATTRIBUTE_APPEND" } diff --git a/shared/src/testFixtures/resources/ngsild/events/authorization/SpecificAccessPolicyAddOnEntity.json b/shared/src/testFixtures/resources/ngsild/events/authorization/SpecificAccessPolicyAddOnEntity.json index 1fd771f3d..624312332 100644 --- a/shared/src/testFixtures/resources/ngsild/events/authorization/SpecificAccessPolicyAddOnEntity.json +++ b/shared/src/testFixtures/resources/ngsild/events/authorization/SpecificAccessPolicyAddOnEntity.json @@ -8,6 +8,5 @@ "contexts": [ "https://easy-global-market.github.io/ngsild-api-data-models/authorization/jsonld-contexts/authorization-compound.jsonld" ], - "overwrite": true, "operationType": "ATTRIBUTE_APPEND" } diff --git a/shared/src/testFixtures/resources/ngsild/events/authorization/UserDeleteEvent.json b/shared/src/testFixtures/resources/ngsild/events/authorization/UserDeleteEvent.json index 918acee3a..349e09b01 100644 --- a/shared/src/testFixtures/resources/ngsild/events/authorization/UserDeleteEvent.json +++ b/shared/src/testFixtures/resources/ngsild/events/authorization/UserDeleteEvent.json @@ -3,6 +3,7 @@ "tenantName": "urn:ngsi-ld:tenant:default", "entityId": "urn:ngsi-ld:User:6ad19fe0-fc11-4024-85f2-931c6fa6f7e0", "entityTypes": ["User"], + "updatedEntity": "{ \"https://uri.etsi.org/ngsi-ld/deletedAt\": [{ \"@type\": \"https://uri.etsi.org/ngsi-ld/DateTime\",\"@value\": \"2024-12-29T00:00:00Z\"} ], \"@id\": \"urn:ngsi-ld:User:6ad19fe0-fc11-4024-85f2-931c6fa6f7e0\" }]", "contexts": [ "https://easy-global-market.github.io/ngsild-api-data-models/authorization/jsonld-contexts/authorization-compound.jsonld" ] diff --git a/shared/src/testFixtures/resources/ngsild/events/entity/attributeDeleteAllInstancesEvent.json b/shared/src/testFixtures/resources/ngsild/events/entity/attributeDeleteAllInstancesEvent.json deleted file mode 100644 index f74eabc89..000000000 --- a/shared/src/testFixtures/resources/ngsild/events/entity/attributeDeleteAllInstancesEvent.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "tenantName": "urn:ngsi-ld:tenant:default", - "entityId" : "urn:ngsi-ld:BeeHive:01", - "entityTypes" : [ "https://ontology.eglobalmark.com/apic#BeeHive" ], - "attributeName" : "https://ontology.eglobalmark.com/apic#temperature", - "updatedEntity" : "{\"@id\":\"urn:ngsi-ld:BeeHive:01\",\"@type\":[\"https://ontology.eglobalmark.com/apic#BeeHive\"],\"https://ontology.eglobalmark.com/apic#humidity\":[{\"@type\":[\"https://uri.etsi.org/ngsi-ld/Property\"],\"https://ontology.eglobalmark.com/egm#observedBy\":[{\"@type\":[\"https://uri.etsi.org/ngsi-ld/Relationship\"],\"https://uri.etsi.org/ngsi-ld/createdAt\":[{\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\",\"@value\":\"2022-02-13T08:02:50.011609Z\"}],\"https://uri.etsi.org/ngsi-ld/hasObject\":[{\"@id\":\"urn:ngsi-ld:Sensor:02\"}]}],\"https://uri.etsi.org/ngsi-ld/createdAt\":[{\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\",\"@value\":\"2022-02-13T08:02:49.982113Z\"}],\"https://uri.etsi.org/ngsi-ld/hasValue\":[{\"@value\":60}],\"https://uri.etsi.org/ngsi-ld/observedAt\":[{\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\",\"@value\":\"2019-10-26T21:32:52.986010Z\"}],\"https://uri.etsi.org/ngsi-ld/unitCode\":[{\"@value\":\"P1\"}]}],\"https://ontology.eglobalmark.com/apic#luminosity\":[{\"@type\":[\"https://uri.etsi.org/ngsi-ld/Property\"],\"https://ontology.eglobalmark.com/egm#observedBy\":[{\"@type\":[\"https://uri.etsi.org/ngsi-ld/Relationship\"],\"https://uri.etsi.org/ngsi-ld/createdAt\":[{\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\",\"@value\":\"2022-02-13T08:02:57.017715Z\"}],\"https://uri.etsi.org/ngsi-ld/hasObject\":[{\"@id\":\"urn:ngsi-ld:Sensor:02\"}]}],\"https://uri.etsi.org/ngsi-ld/createdAt\":[{\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\",\"@value\":\"2022-02-13T08:02:56.987108Z\"}],\"https://uri.etsi.org/ngsi-ld/hasValue\":[{\"@value\":120}],\"https://uri.etsi.org/ngsi-ld/observedAt\":[{\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\",\"@value\":\"2022-02-02T21:32:52.986010Z\"}],\"https://uri.etsi.org/ngsi-ld/unitCode\":[{\"@value\":\"LUX\"}]}],\"https://ontology.eglobalmark.com/egm#belongs\":[{\"@type\":[\"https://uri.etsi.org/ngsi-ld/Relationship\"],\"https://uri.etsi.org/ngsi-ld/createdAt\":[{\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\",\"@value\":\"2022-02-13T08:02:49.793453Z\"}],\"https://uri.etsi.org/ngsi-ld/hasObject\":[{\"@id\":\"urn:ngsi-ld:Apiary:01\"}]}],\"https://ontology.eglobalmark.com/egm#createdBy\":[{\"@type\":[\"https://uri.etsi.org/ngsi-ld/Relationship\"],\"https://uri.etsi.org/ngsi-ld/createdAt\":[{\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\",\"@value\":\"2022-02-13T08:07:08.429628Z\"}],\"https://uri.etsi.org/ngsi-ld/hasObject\":[{\"@id\":\"urn:ngsi-ld:Craftman:01\"}]}],\"https://ontology.eglobalmark.com/egm#managedBy\":[{\"@type\":[\"https://uri.etsi.org/ngsi-ld/Relationship\"],\"https://uri.etsi.org/ngsi-ld/createdAt\":[{\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\",\"@value\":\"2022-02-13T14:00:45.114035Z\"}],\"https://uri.etsi.org/ngsi-ld/hasObject\":[{\"@id\":\"urn:ngsi-ld:Beekeeper:02\"}],\"https://uri.etsi.org/ngsi-ld/modifiedAt\":[{\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\",\"@value\":\"2022-02-13T14:10:16.963869Z\"}]}],\"https://schema.org/name\":[{\"@type\":[\"https://uri.etsi.org/ngsi-ld/Property\"],\"https://uri.etsi.org/ngsi-ld/createdAt\":[{\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\",\"@value\":\"2022-02-13T14:14:03.758379Z\"}],\"https://uri.etsi.org/ngsi-ld/hasValue\":[{\"@value\":\"Beehive - Biot\"}]}],\"https://uri.etsi.org/ngsi-ld/createdAt\":[{\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\",\"@value\":\"2022-02-13T08:02:49.359316Z\"}],\"https://uri.etsi.org/ngsi-ld/location\":[{\"@type\":[\"https://uri.etsi.org/ngsi-ld/GeoProperty\"],\"https://uri.etsi.org/ngsi-ld/hasValue\":[{\"@value\":\"POINT (24.30623 60.07966)\"}]}],\"https://uri.etsi.org/ngsi-ld/modifiedAt\":[{\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\",\"@value\":\"2022-02-13T14:14:03.812339Z\"}]}", - "contexts" : [], - "operationType" : "ATTRIBUTE_DELETE_ALL_INSTANCES" -} diff --git a/shared/src/testFixtures/resources/ngsild/events/entity/entityDeleteEvent.json b/shared/src/testFixtures/resources/ngsild/events/entity/entityDeleteEvent.json index ed92fc60c..5b381132a 100644 --- a/shared/src/testFixtures/resources/ngsild/events/entity/entityDeleteEvent.json +++ b/shared/src/testFixtures/resources/ngsild/events/entity/entityDeleteEvent.json @@ -3,7 +3,8 @@ "tenantName": "urn:ngsi-ld:tenant:default", "entityId" : "urn:ngsi-ld:BeeHive:01", "entityTypes" : [ "https://ontology.eglobalmark.com/apic#BeeHive" ], - "deletedEntity":"{\"@id\":\"urn:ngsi-ld:BeeHive:01\",\"@type\":[\"https://ontology.eglobalmark.com/apic#BeeHive\"],\"https://ontology.eglobalmark.com/apic#humidity\":[{\"@type\":[\"https://uri.etsi.org/ngsi-ld/Property\"],\"https://ontology.eglobalmark.com/egm#observedBy\":[{\"@type\":[\"https://uri.etsi.org/ngsi-ld/Relationship\"],\"https://uri.etsi.org/ngsi-ld/createdAt\":[{\"@value\":\"2022-02-12T08:36:59.455870Z\",\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\"}],\"https://uri.etsi.org/ngsi-ld/hasObject\":[{\"@id\":\"urn:ngsi-ld:Sensor:02\"}]}],\"https://uri.etsi.org/ngsi-ld/createdAt\":[{\"@value\":\"2022-02-12T08:36:59.448205Z\",\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\"}],\"https://uri.etsi.org/ngsi-ld/hasValue\":[{\"@value\":60}],\"https://uri.etsi.org/ngsi-ld/observedAt\":[{\"@value\":\"2019-10-26T21:32:52.986010Z\",\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\"}],\"https://uri.etsi.org/ngsi-ld/unitCode\":[{\"@value\":\"P1\"}]}],\"https://ontology.eglobalmark.com/apic#temperature\":[{\"@type\":[\"https://uri.etsi.org/ngsi-ld/Property\"],\"https://ontology.eglobalmark.com/egm#observedBy\":[{\"@type\":[\"https://uri.etsi.org/ngsi-ld/Relationship\"],\"https://uri.etsi.org/ngsi-ld/createdAt\":[{\"@value\":\"2022-02-12T08:36:59.473904Z\",\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\"}],\"https://uri.etsi.org/ngsi-ld/hasObject\":[{\"@id\":\"urn:ngsi-ld:Sensor:01\"}]}],\"https://uri.etsi.org/ngsi-ld/createdAt\":[{\"@value\":\"2022-02-12T08:36:59.465937Z\",\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\"}],\"https://uri.etsi.org/ngsi-ld/hasValue\":[{\"@value\":22.2}],\"https://uri.etsi.org/ngsi-ld/observedAt\":[{\"@value\":\"2019-10-26T21:32:52.986010Z\",\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\"}],\"https://uri.etsi.org/ngsi-ld/unitCode\":[{\"@value\":\"CEL\"}]}],\"https://ontology.eglobalmark.com/egm#belongs\":[{\"@type\":[\"https://uri.etsi.org/ngsi-ld/Relationship\"],\"https://uri.etsi.org/ngsi-ld/createdAt\":[{\"@value\":\"2022-02-12T08:36:59.389815Z\",\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\"}],\"https://uri.etsi.org/ngsi-ld/hasObject\":[{\"@id\":\"urn:ngsi-ld:Apiary:01\"}]}],\"https://ontology.eglobalmark.com/egm#managedBy\":[{\"@type\":[\"https://uri.etsi.org/ngsi-ld/Relationship\"],\"https://uri.etsi.org/ngsi-ld/createdAt\":[{\"@value\":\"2022-02-12T08:36:59.417938Z\",\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\"}],\"https://uri.etsi.org/ngsi-ld/hasObject\":[{\"@id\":\"urn:ngsi-ld:Beekeeper:01\"}]}],\"https://uri.etsi.org/ngsi-ld/createdAt\":[{\"@value\":\"2022-02-12T08:36:59.179446Z\",\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\"}],\"https://uri.etsi.org/ngsi-ld/location\":[{\"@type\":[\"https://uri.etsi.org/ngsi-ld/GeoProperty\"],\"https://uri.etsi.org/ngsi-ld/hasValue\":[{\"@value\":\"POINT (24.30623 60.07966)\"}]}],\"https://uri.etsi.org/ngsi-ld/modifiedAt\":[{\"@value\":\"2022-02-12T08:36:59.218595Z\",\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\"}]}", + "previousEntity":"{\"@id\":\"urn:ngsi-ld:BeeHive:01\",\"@type\":[\"https://ontology.eglobalmark.com/apic#BeeHive\"],\"https://ontology.eglobalmark.com/apic#humidity\":[{\"@type\":[\"https://uri.etsi.org/ngsi-ld/Property\"],\"https://ontology.eglobalmark.com/egm#observedBy\":[{\"@type\":[\"https://uri.etsi.org/ngsi-ld/Relationship\"],\"https://uri.etsi.org/ngsi-ld/createdAt\":[{\"@value\":\"2022-02-12T08:36:59.455870Z\",\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\"}],\"https://uri.etsi.org/ngsi-ld/hasObject\":[{\"@id\":\"urn:ngsi-ld:Sensor:02\"}]}],\"https://uri.etsi.org/ngsi-ld/createdAt\":[{\"@value\":\"2022-02-12T08:36:59.448205Z\",\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\"}],\"https://uri.etsi.org/ngsi-ld/hasValue\":[{\"@value\":60}],\"https://uri.etsi.org/ngsi-ld/observedAt\":[{\"@value\":\"2019-10-26T21:32:52.986010Z\",\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\"}],\"https://uri.etsi.org/ngsi-ld/unitCode\":[{\"@value\":\"P1\"}]}],\"https://ontology.eglobalmark.com/apic#temperature\":[{\"@type\":[\"https://uri.etsi.org/ngsi-ld/Property\"],\"https://ontology.eglobalmark.com/egm#observedBy\":[{\"@type\":[\"https://uri.etsi.org/ngsi-ld/Relationship\"],\"https://uri.etsi.org/ngsi-ld/createdAt\":[{\"@value\":\"2022-02-12T08:36:59.473904Z\",\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\"}],\"https://uri.etsi.org/ngsi-ld/hasObject\":[{\"@id\":\"urn:ngsi-ld:Sensor:01\"}]}],\"https://uri.etsi.org/ngsi-ld/createdAt\":[{\"@value\":\"2022-02-12T08:36:59.465937Z\",\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\"}],\"https://uri.etsi.org/ngsi-ld/hasValue\":[{\"@value\":22.2}],\"https://uri.etsi.org/ngsi-ld/observedAt\":[{\"@value\":\"2019-10-26T21:32:52.986010Z\",\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\"}],\"https://uri.etsi.org/ngsi-ld/unitCode\":[{\"@value\":\"CEL\"}]}],\"https://ontology.eglobalmark.com/egm#belongs\":[{\"@type\":[\"https://uri.etsi.org/ngsi-ld/Relationship\"],\"https://uri.etsi.org/ngsi-ld/createdAt\":[{\"@value\":\"2022-02-12T08:36:59.389815Z\",\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\"}],\"https://uri.etsi.org/ngsi-ld/hasObject\":[{\"@id\":\"urn:ngsi-ld:Apiary:01\"}]}],\"https://ontology.eglobalmark.com/egm#managedBy\":[{\"@type\":[\"https://uri.etsi.org/ngsi-ld/Relationship\"],\"https://uri.etsi.org/ngsi-ld/createdAt\":[{\"@value\":\"2022-02-12T08:36:59.417938Z\",\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\"}],\"https://uri.etsi.org/ngsi-ld/hasObject\":[{\"@id\":\"urn:ngsi-ld:Beekeeper:01\"}]}],\"https://uri.etsi.org/ngsi-ld/createdAt\":[{\"@value\":\"2022-02-12T08:36:59.179446Z\",\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\"}],\"https://uri.etsi.org/ngsi-ld/location\":[{\"@type\":[\"https://uri.etsi.org/ngsi-ld/GeoProperty\"],\"https://uri.etsi.org/ngsi-ld/hasValue\":[{\"@value\":\"POINT (24.30623 60.07966)\"}]}],\"https://uri.etsi.org/ngsi-ld/modifiedAt\":[{\"@value\":\"2022-02-12T08:36:59.218595Z\",\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\"}]}", + "updatedEntity":"{\"@id\":\"urn:ngsi-ld:BeeHive:01\",\"https://uri.etsi.org/ngsi-ld/deletedAt\":[{\"@value\":\"2024-12-23T17:01:02Z\",\"@type\":\"https://uri.etsi.org/ngsi-ld/DateTime\"}]}", "contexts" : [], "operationType" : "ENTITY_DELETE" } diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/listener/EntityEventListenerService.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/listener/EntityEventListenerService.kt index d4891cd73..d815ec65e 100644 --- a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/listener/EntityEventListenerService.kt +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/listener/EntityEventListenerService.kt @@ -4,7 +4,6 @@ import arrow.core.Either import arrow.core.raise.either import com.egm.stellio.shared.model.APIException import com.egm.stellio.shared.model.AttributeAppendEvent -import com.egm.stellio.shared.model.AttributeDeleteAllInstancesEvent import com.egm.stellio.shared.model.AttributeDeleteEvent import com.egm.stellio.shared.model.AttributeReplaceEvent import com.egm.stellio.shared.model.AttributeUpdateEvent @@ -19,7 +18,6 @@ import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_EXPANDED_ENTITY_CORE_MEMBE import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_SYSATTRS_PROPERTIES import com.egm.stellio.shared.util.JsonUtils.deserializeAs import com.egm.stellio.shared.util.JsonUtils.deserializeAsMap -import com.egm.stellio.shared.util.JsonUtils.serializeObject import com.egm.stellio.shared.web.NGSILD_TENANT_HEADER import com.egm.stellio.subscription.model.NotificationTrigger import com.egm.stellio.subscription.service.NotificationService @@ -63,51 +61,45 @@ class EntityEventListenerService( is EntityCreateEvent -> handleEntityEvent( tenantName, entityEvent.operationPayload.getUpdatedAttributes(), - entityEvent.getEntity(), + Pair(entityEvent.getEntity(), entityEvent.getEntity()), NotificationTrigger.ENTITY_CREATED ) is EntityReplaceEvent -> entityEvent.operationPayload.getUpdatedAttributes().forEach { attribute -> handleEntityEvent( tenantName, setOf(attribute), - entityEvent.getEntity(), + Pair(entityEvent.getEntity(), entityEvent.getEntity()), NotificationTrigger.ATTRIBUTE_CREATED ) } is EntityDeleteEvent -> handleEntityEvent( tenantName, - entityEvent.deletedEntity?.getUpdatedAttributes() ?: emptySet(), - entityEvent.getEntity() ?: serializeObject(emptyMap()), + emptySet(), + Pair(entityEvent.getEntity()!!, entityEvent.updatedEntity), NotificationTrigger.ENTITY_DELETED ) is AttributeAppendEvent -> handleEntityEvent( tenantName, setOf(entityEvent.attributeName), - entityEvent.getEntity(), + Pair(entityEvent.getEntity(), entityEvent.getEntity()), NotificationTrigger.ATTRIBUTE_CREATED ) is AttributeReplaceEvent -> handleEntityEvent( tenantName, setOf(entityEvent.attributeName), - entityEvent.getEntity(), + Pair(entityEvent.getEntity(), entityEvent.getEntity()), NotificationTrigger.ATTRIBUTE_UPDATED ) is AttributeUpdateEvent -> handleEntityEvent( tenantName, setOf(entityEvent.attributeName), - entityEvent.getEntity(), + Pair(entityEvent.getEntity(), entityEvent.getEntity()), NotificationTrigger.ATTRIBUTE_UPDATED ) is AttributeDeleteEvent -> handleEntityEvent( tenantName, setOf(entityEvent.attributeName), - entityEvent.getEntity(), - NotificationTrigger.ATTRIBUTE_DELETED - ) - is AttributeDeleteAllInstancesEvent -> handleEntityEvent( - tenantName, - setOf(entityEvent.attributeName), - entityEvent.getEntity(), + Pair(entityEvent.getEntity(), entityEvent.getEntity()), NotificationTrigger.ATTRIBUTE_DELETED ) } @@ -119,14 +111,15 @@ class EntityEventListenerService( private suspend fun handleEntityEvent( tenantName: String, updatedAttributes: Set, - entityPayload: String, + previousAndUpdatedPayloads: Pair, notificationTrigger: NotificationTrigger ): Either = either { logger.debug("Attributes considered in the event: {}", updatedAttributes) - val expandedEntity = ExpandedEntity(entityPayload.deserializeAsMap()) + val expandedEntityForMatching = ExpandedEntity(previousAndUpdatedPayloads.first.deserializeAsMap()) + val expandedEntityForNotification = ExpandedEntity(previousAndUpdatedPayloads.second.deserializeAsMap()) mono { notificationService.notifyMatchingSubscribers( - expandedEntity, + Pair(expandedEntityForMatching, expandedEntityForNotification), updatedAttributes, notificationTrigger ) @@ -139,10 +132,8 @@ class EntityEventListenerService( else logger.error("Error when trying to notifiy subscribers: {}", it.message, it) }, { results -> - val totalNotifications = results.size - val succeeded = results.count { it.third } - val failed = results.count { !it.third } - logger.debug("Notified $totalNotifications subscribers (success : $succeeded / failure : $failed)") + val (succeeded, failed) = results.partition { it.third }.let { Pair(it.first.size, it.second.size) } + logger.debug("Notified ${succeeded + failed} subscribers (success : $succeeded / failure : $failed)") }) } } diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/NotificationService.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/NotificationService.kt index 1a054858c..ddda29ca1 100644 --- a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/NotificationService.kt +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/service/NotificationService.kt @@ -38,13 +38,17 @@ class NotificationService( private val logger = LoggerFactory.getLogger(javaClass) suspend fun notifyMatchingSubscribers( - expandedEntity: ExpandedEntity, + previousAndUpdatedExpandedEntities: Pair, updatedAttributes: Set, notificationTrigger: NotificationTrigger ): Either>> = either { - subscriptionService.getMatchingSubscriptions(expandedEntity, updatedAttributes, notificationTrigger).bind() + subscriptionService.getMatchingSubscriptions( + previousAndUpdatedExpandedEntities.first, + updatedAttributes, + notificationTrigger + ).bind() .map { - val filteredEntity = expandedEntity.filterAttributes( + val filteredEntity = previousAndUpdatedExpandedEntities.second.filterAttributes( it.notification.attributes?.toSet().orEmpty(), it.datasetId?.toSet().orEmpty() ) diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/NotificationServiceTests.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/NotificationServiceTests.kt index 90ea23eee..83dc4fd46 100644 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/NotificationServiceTests.kt +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/NotificationServiceTests.kt @@ -136,7 +136,7 @@ class NotificationServiceTests { ) notificationService.notifyMatchingSubscribers( - expandedEntity, + Pair(expandedEntity, expandedEntity), setOf(NGSILD_NAME_PROPERTY), ATTRIBUTE_UPDATED ).shouldSucceedWith { @@ -179,7 +179,7 @@ class NotificationServiceTests { ) notificationService.notifyMatchingSubscribers( - expandedEntity, + Pair(expandedEntity, expandedEntity), setOf(NGSILD_NAME_PROPERTY), ATTRIBUTE_UPDATED ).shouldSucceedWith { @@ -224,7 +224,7 @@ class NotificationServiceTests { ) notificationService.notifyMatchingSubscribers( - expandedEntity, + Pair(expandedEntity, expandedEntity), setOf(NGSILD_NAME_PROPERTY), ATTRIBUTE_UPDATED ).shouldSucceedWith { notificationResults -> @@ -265,7 +265,7 @@ class NotificationServiceTests { ) notificationService.notifyMatchingSubscribers( - expandedEntity, + Pair(expandedEntity, expandedEntity), setOf(NGSILD_NAME_TERM), ATTRIBUTE_UPDATED ).shouldSucceedWith { notificationResults -> @@ -295,7 +295,7 @@ class NotificationServiceTests { ) notificationService.notifyMatchingSubscribers( - expandedEntity, + Pair(expandedEntity, expandedEntity), setOf(NGSILD_NAME_PROPERTY), ATTRIBUTE_UPDATED ).shouldSucceedWith { @@ -333,7 +333,7 @@ class NotificationServiceTests { ) notificationService.notifyMatchingSubscribers( - expandedEntity, + Pair(expandedEntity, expandedEntity), setOf(NGSILD_NAME_PROPERTY), ATTRIBUTE_DELETED ).shouldSucceedWith { @@ -379,7 +379,7 @@ class NotificationServiceTests { ) notificationService.notifyMatchingSubscribers( - expandedEntity, + Pair(expandedEntity, expandedEntity), setOf(NGSILD_NAME_PROPERTY), ATTRIBUTE_CREATED ).shouldSucceedWith { results -> @@ -540,7 +540,7 @@ class NotificationServiceTests { ) notificationService.notifyMatchingSubscribers( - expandedEntity, + Pair(expandedEntity, expandedEntity), setOf(NGSILD_NAME_PROPERTY), ATTRIBUTE_UPDATED ).shouldSucceedWith { @@ -582,7 +582,7 @@ class NotificationServiceTests { ) notificationService.notifyMatchingSubscribers( - expandedEntity, + Pair(expandedEntity, expandedEntity), setOf(NGSILD_NAME_PROPERTY), ATTRIBUTE_UPDATED ).shouldSucceedWith { @@ -641,7 +641,7 @@ class NotificationServiceTests { ) notificationService.notifyMatchingSubscribers( - expandedEntity, + Pair(expandedEntity, expandedEntity), setOf(FRIENDLYNAME_LANGUAGEPROPERTY), ATTRIBUTE_UPDATED ).shouldSucceedWith { diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/SubscriptionServiceTests.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/SubscriptionServiceTests.kt index 2c2ca9709..225e1d829 100644 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/SubscriptionServiceTests.kt +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/service/SubscriptionServiceTests.kt @@ -70,13 +70,13 @@ import org.springframework.test.context.ActiveProfiles import org.springframework.test.context.TestPropertySource import java.net.URI import java.time.ZonedDateTime -import java.util.UUID +import java.util.* import kotlin.time.Duration @SpringBootTest @ActiveProfiles("test") @TestPropertySource(properties = ["application.authentication.enabled=false"]) -class SubscriptionServiceTests : WithTimescaleContainer, WithKafkaContainer { +class SubscriptionServiceTests : WithTimescaleContainer, WithKafkaContainer() { @Autowired private lateinit var subscriptionService: SubscriptionService @@ -347,8 +347,8 @@ class SubscriptionServiceTests : WithTimescaleContainer, WithKafkaContainer { it.notification.endpoint.uri == URI("http://localhost:8084") && it.notification.endpoint.accept == Endpoint.AcceptType.JSON && it.entities != null && - it.entities!!.size == 1 && - it.entities!!.all { entitySelector -> entitySelector.typeSelection == BEEHIVE_TYPE } && + it.entities.size == 1 && + it.entities.all { entitySelector -> entitySelector.typeSelection == BEEHIVE_TYPE } && it.watchedAttributes == null && it.isActive } @@ -383,10 +383,10 @@ class SubscriptionServiceTests : WithTimescaleContainer, WithKafkaContainer { it.id == "urn:ngsi-ld:Subscription:1".toUri() && it.subscriptionName == "A subscription with all possible members" && it.description == "A possible description" && it.entities != null && - it.entities!!.size == 3 && - it.entities!!.all { it.typeSelection == BEEHIVE_TYPE } && - it.entities!!.any { it.id == "urn:ngsi-ld:Beehive:1234567890".toUri() } && - it.entities!!.any { it.idPattern == "urn:ngsi-ld:Beehive:1234*" } && + it.entities.size == 3 && + it.entities.all { it.typeSelection == BEEHIVE_TYPE } && + it.entities.any { it.id == "urn:ngsi-ld:Beehive:1234567890".toUri() } && + it.entities.any { it.idPattern == "urn:ngsi-ld:Beehive:1234*" } && it.watchedAttributes == listOf(INCOMING_PROPERTY) && it.notificationTrigger == listOf( ENTITY_CREATED.notificationTrigger, @@ -395,11 +395,11 @@ class SubscriptionServiceTests : WithTimescaleContainer, WithKafkaContainer { ) && it.timeInterval == null && it.q == "foodQuantity<150;foodName=='dietary fibres'" && it.geoQ != null && - it.geoQ!!.georel == "within" && - it.geoQ!!.geometry == "Polygon" && - it.geoQ!!.coordinates == + it.geoQ.georel == "within" && + it.geoQ.geometry == "Polygon" && + it.geoQ.coordinates == "[[[100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0]]]" && - it.geoQ!!.geoproperty == NGSILD_LOCATION_PROPERTY && + it.geoQ.geoproperty == NGSILD_LOCATION_PROPERTY && it.scopeQ == "/Nantes/+" && it.notification.attributes == listOf(INCOMING_PROPERTY, OUTGOING_PROPERTY) && it.notification.format == FormatType.NORMALIZED && @@ -438,9 +438,9 @@ class SubscriptionServiceTests : WithTimescaleContainer, WithKafkaContainer { assertThat(persistedSubscription) .matches { it.notification.lastNotification != null && - it.notification.lastNotification!!.isEqual(notifiedAt) && + it.notification.lastNotification.isEqual(notifiedAt) && it.notification.lastSuccess != null && - it.notification.lastSuccess!!.isEqual(notifiedAt) && + it.notification.lastSuccess.isEqual(notifiedAt) && it.notification.lastFailure == null && it.notification.timesSent == 1 && it.notification.status == StatusType.OK @@ -788,6 +788,27 @@ class SubscriptionServiceTests : WithTimescaleContainer, WithKafkaContainer { } } + @Test + fun `it should retrieve a subscription with entityDeleted trigger matched with an entity delete event`() = + runTest { + val subscription = gimmeSubscriptionFromMembers( + mapOf( + "entities" to listOf(mapOf("type" to BEEHIVE_COMPACT_TYPE)), + "notificationTrigger" to listOf(ENTITY_DELETED.notificationTrigger) + ) + ) + subscriptionService.create(subscription, mockUserSub).shouldSucceed() + + val expandedEntity = loadAndExpandMinimalEntity("urn:ngsi-ld:Beehive:1234567890", BEEHIVE_COMPACT_TYPE) + subscriptionService.getMatchingSubscriptions( + expandedEntity, + emptySet(), + ENTITY_DELETED + ).shouldSucceedWith { + assertEquals(1, it.size) + } + } + @Test fun `it should update a subscription`() = runTest { val subscription = loadAndDeserializeSubscription("subscription_minimal_entities.json") @@ -821,9 +842,9 @@ class SubscriptionServiceTests : WithTimescaleContainer, WithKafkaContainer { it.watchedAttributes!! == listOf(INCOMING_PROPERTY, TEMPERATURE_PROPERTY) && it.scopeQ == "/A/#,/B" && it.geoQ!!.georel == "equals" && - it.geoQ!!.geometry == "Point" && - it.geoQ!!.coordinates == "[100.0, 0.0]" && - it.geoQ!!.geoproperty == "https://uri.etsi.org/ngsi-ld/observationSpace" && + it.geoQ.geometry == "Point" && + it.geoQ.coordinates == "[100.0, 0.0]" && + it.geoQ.geoproperty == "https://uri.etsi.org/ngsi-ld/observationSpace" && it.throttling == 50 && it.lang == "fr-CH,fr" } @@ -883,21 +904,21 @@ class SubscriptionServiceTests : WithTimescaleContainer, WithKafkaContainer { assertThat(updatedSubscription) .matches { it.entities != null && - it.entities!!.contains( + it.entities.contains( EntitySelector( id = "urn:ngsi-ld:Beehive:123".toUri(), idPattern = null, typeSelection = BEEHIVE_TYPE ) ) && - it.entities!!.contains( + it.entities.contains( EntitySelector( id = null, idPattern = "urn:ngsi-ld:Beehive:12*", typeSelection = BEEHIVE_TYPE ) ) && - it.entities!!.size == 2 + it.entities.size == 2 } } diff --git a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithKafkaContainer.kt b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithKafkaContainer.kt index cbe761176..2b40e471c 100644 --- a/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithKafkaContainer.kt +++ b/subscription-service/src/test/kotlin/com/egm/stellio/subscription/support/WithKafkaContainer.kt @@ -1,29 +1,21 @@ package com.egm.stellio.subscription.support -import org.springframework.test.context.DynamicPropertyRegistry -import org.springframework.test.context.DynamicPropertySource +import org.springframework.boot.testcontainers.service.connection.ServiceConnection +import org.testcontainers.junit.jupiter.Container +import org.testcontainers.junit.jupiter.Testcontainers import org.testcontainers.kafka.ConfluentKafkaContainer -import org.testcontainers.utility.DockerImageName -interface WithKafkaContainer { +@Testcontainers +@Suppress("UtilityClassWithPublicConstructor") +open class WithKafkaContainer { companion object { - private val kafkaImage: DockerImageName = - DockerImageName.parse("confluentinc/cp-kafka:7.6.0") - - private val kafkaContainer = ConfluentKafkaContainer(kafkaImage).apply { - withReuse(true) - } - + @Container + @ServiceConnection @JvmStatic - @DynamicPropertySource - fun properties(registry: DynamicPropertyRegistry) { - registry.add("spring.kafka.bootstrap-servers") { kafkaContainer.bootstrapServers } - } - - init { - kafkaContainer.start() + val kafkaContainer = ConfluentKafkaContainer("confluentinc/cp-kafka:7.6.0").apply { + withReuse(true) } } }