From fec52366d8c24f9431ae69e622cd0eb9998579ef Mon Sep 17 00:00:00 2001 From: Oliver <20188437+olivergrabinski@users.noreply.github.com> Date: Fri, 26 Apr 2024 09:22:02 +0200 Subject: [PATCH] Allow disabling of resource validation in nexus ship (#4908) --- ship/src/main/resources/ship-default.conf | 3 + .../epfl/bluebrain/nexus/ship/RunShip.scala | 2 +- .../nexus/ship/config/InputConfig.scala | 3 +- .../nexus/ship/resources/ResourceWiring.scala | 65 +++++++++++++++++-- .../ship/config/ShipConfigFixtures.scala | 3 +- 5 files changed, 66 insertions(+), 10 deletions(-) diff --git a/ship/src/main/resources/ship-default.conf b/ship/src/main/resources/ship-default.conf index 4996464caa..3289e7fba7 100644 --- a/ship/src/main/resources/ship-default.conf +++ b/ship/src/main/resources/ship-default.conf @@ -79,6 +79,9 @@ ship { # The bucket to which the files will be copied by the Nexus Ship target-bucket = "nexus-delta-production" + # If true, no resource validation is performed + disable-resource-validation = false + storages { # S3 compatible storage configuration diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala index ba903443cb..32b9b68347 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala @@ -56,7 +56,7 @@ object RunShip { remoteContextResolution <- ContextWiring.remoteContextResolution validateShacl <- ValidateShacl(remoteContextResolution) (schemaLog, fetchSchema) = SchemaWiring(config.eventLog, eventClock, xas) - (resourceLog, fetchResource) = ResourceWiring(fetchContext, fetchSchema, validateShacl, eventLogConfig, eventClock, xas) + (resourceLog, fetchResource) = ResourceWiring(fetchContext, fetchSchema, validateShacl, config, eventClock, xas) rcr = ContextWiring.resolverContextResolution(fetchResource, fetchContext, remoteContextResolution, eventLogConfig, eventClock, xas) schemaImports = SchemaWiring.schemaImports(fetchResource, fetchSchema, fetchContext, eventLogConfig, eventClock, xas) // Processors diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/InputConfig.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/InputConfig.scala index e72af35b5a..04dd236f96 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/InputConfig.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/InputConfig.scala @@ -24,7 +24,8 @@ final case class InputConfig( serviceAccount: ServiceAccountConfig, storages: StoragesConfig, importBucket: BucketName, - targetBucket: BucketName + targetBucket: BucketName, + disableResourceValidation: Boolean ) object InputConfig { diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resources/ResourceWiring.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resources/ResourceWiring.scala index fbf112cd07..4143516292 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resources/ResourceWiring.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resources/ResourceWiring.scala @@ -1,17 +1,26 @@ package ch.epfl.bluebrain.nexus.ship.resources +import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi -import ch.epfl.bluebrain.nexus.delta.rdf.shacl.ValidateShacl +import ch.epfl.bluebrain.nexus.delta.rdf.shacl.{ValidateShacl, ValidationReport} +import ch.epfl.bluebrain.nexus.delta.sdk.jsonld.JsonLdAssembly +import ch.epfl.bluebrain.nexus.delta.sdk.model.ResourceF import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResourceResolution import ch.epfl.bluebrain.nexus.delta.sdk.resources.Resources.ResourceLog -import ch.epfl.bluebrain.nexus.delta.sdk.resources.{DetectChange, FetchResource, Resources, ValidateResource} +import ch.epfl.bluebrain.nexus.delta.sdk.resources.SchemaClaim.DefinedSchemaClaim +import ch.epfl.bluebrain.nexus.delta.sdk.resources.ValidationResult.{NoValidation, Validated} +import ch.epfl.bluebrain.nexus.delta.sdk.resources._ import ch.epfl.bluebrain.nexus.delta.sdk.schemas.FetchSchema -import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig +import ch.epfl.bluebrain.nexus.delta.sdk.schemas.model.Schema +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ResourceRef import ch.epfl.bluebrain.nexus.delta.sourcing.{ScopedEventLog, Transactors} import ch.epfl.bluebrain.nexus.ship.EventClock import ch.epfl.bluebrain.nexus.ship.acls.AclWiring.alwaysAuthorize +import ch.epfl.bluebrain.nexus.ship.config.InputConfig import ch.epfl.bluebrain.nexus.ship.resolvers.ResolverWiring +import io.circe.Json +import io.circe.syntax.KeyOps object ResourceWiring { @@ -19,21 +28,63 @@ object ResourceWiring { fetchContext: FetchContext, fetchSchema: FetchSchema, validateShacl: ValidateShacl, - config: EventLogConfig, + config: InputConfig, clock: EventClock, xas: Transactors )(implicit jsonLdApi: JsonLdApi ): (ResourceLog, FetchResource) = { val detectChange = DetectChange(false) - val resolvers = ResolverWiring.resolvers(fetchContext, config, clock, xas) + val resolvers = ResolverWiring.resolvers(fetchContext, config.eventLog, clock, xas) val resourceResolution = ResourceResolution.schemaResource(alwaysAuthorize, resolvers, fetchSchema, excludeDeprecated = false) val validate = ValidateResource(resourceResolution, validateShacl) - val resourceDef = Resources.definition(validate, detectChange, clock) - val log = ScopedEventLog(resourceDef, config, xas) + val validation = if (config.disableResourceValidation) alwaysValidateResource(fetchSchema) else validate + val resourceDef = Resources.definition(validation, detectChange, clock) + + val log = ScopedEventLog(resourceDef, config.eventLog, xas) (log, FetchResource(log)) } + private def alwaysValidateResource(fetchSchema: FetchSchema): ValidateResource = new ValidateResource { + val defaultReport: ValidationReport = ValidationReport(conforms = true, 5, Json.obj("conforms" := "true")) + + override def apply(jsonld: JsonLdAssembly, schema: SchemaClaim, enforceSchema: Boolean): IO[ValidationResult] = + schema match { + case defined: DefinedSchemaClaim => + fetchSchema.fetch(defined.schemaRef, schema.project).flatMap { + case Some(value) => + IO.pure { + Validated( + schema.project, + ResourceRef.Revision(value.id, value.rev), + defaultReport + ) + } + case None => + IO.pure( + Validated( + schema.project, + ResourceRef.Revision(defined.schemaRef.iri, 1), + defaultReport + ) + ) + } + case other => IO.pure(NoValidation(other.project)) + } + + override def apply( + jsonld: JsonLdAssembly, + schema: ResourceF[Schema] + ): IO[ValidationResult] = + IO.pure( + Validated( + schema.value.project, + ResourceRef.Revision(schema.id, schema.rev), + defaultReport + ) + ) + } + } diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigFixtures.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigFixtures.scala index 2a6176f5cc..9aa854cb05 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigFixtures.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigFixtures.scala @@ -58,7 +58,8 @@ trait ShipConfigFixtures extends ConfigFixtures with StorageFixtures with Classp serviceAccount, StoragesConfig(eventLogConfig, pagination, config.copy(amazon = Some(amazonConfig))), importBucket, - targetBucket + targetBucket, + disableResourceValidation = false ) }