From 090a78edbf95419102f9cc5f3c5b6f496c07f7ff Mon Sep 17 00:00:00 2001 From: Simon Date: Tue, 23 Apr 2024 10:18:28 +0200 Subject: [PATCH] Patch distributions in import batch (#4880) * Patch distributions in import batch * Address feedbacks --------- Co-authored-by: Simon Dumas --- build.sbt | 2 +- .../plugins/archive/ArchiveDownload.scala | 3 +- .../plugins/archive/ArchivePluginModule.scala | 1 + .../archive/model/ArchiveRejection.scala | 2 +- .../plugins/archive/ArchiveDownloadSpec.scala | 4 +- .../plugins/archive/ArchiveRoutesSpec.scala | 2 +- .../delta/plugins/storage}/FileSelf.scala | 4 +- .../storage/files}/FileSelfSuite.scala | 5 +- .../scripts/postgres/drop/drop-tables.ddl | 1 + ...M09_001__ship_original_project_context.ddl | 7 ++ ship/src/main/resources/ship-default.conf | 3 +- .../bluebrain/nexus/ship/ContextWiring.scala | 2 +- .../epfl/bluebrain/nexus/ship/RunShip.scala | 22 ++-- .../nexus/ship/config/InputConfig.scala | 3 +- .../projects/OriginalProjectContext.scala | 49 ++++++++ .../ship/projects/ProjectProcessor.scala | 18 ++- .../ship/resources/DistributionPatcher.scala | 57 +++++++++ .../ship/resources/ResourceProcessor.scala | 19 ++- ship/src/test/resources/config/external.conf | 2 +- .../ship/config/ShipConfigFixtures.scala | 1 + .../nexus/ship/config/ShipConfigSuite.scala | 6 +- .../OriginalProjectContextSuite.scala | 37 ++++++ .../resources/DistributionPatcherSuite.scala | 111 ++++++++++++++++++ 23 files changed, 325 insertions(+), 36 deletions(-) rename delta/plugins/{archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive => storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage}/FileSelf.scala (97%) rename delta/plugins/{archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive => storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files}/FileSelfSuite.scala (94%) create mode 100644 delta/sourcing-psql/src/main/resources/scripts/postgres/init/V1_10_M09_001__ship_original_project_context.ddl create mode 100644 ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/OriginalProjectContext.scala create mode 100644 ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resources/DistributionPatcher.scala create mode 100644 ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/projects/OriginalProjectContextSuite.scala create mode 100644 ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/resources/DistributionPatcherSuite.scala diff --git a/build.sbt b/build.sbt index ab30d7a8de..a5dce380fd 100755 --- a/build.sbt +++ b/build.sbt @@ -754,7 +754,7 @@ lazy val ship = project tests % "test->compile;test->test" ) .settings( - libraryDependencies ++= Seq(declineEffect, logback, fs2Aws, fs2AwsS3), + libraryDependencies ++= Seq(declineEffect, logback, circeOptics), addCompilerPlugin(betterMonadicFor), run / fork := true, buildInfoKeys := Seq[BuildInfoKey](version), diff --git a/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownload.scala b/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownload.scala index 4843fe37e5..30f8c33f2c 100644 --- a/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownload.scala +++ b/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownload.scala @@ -7,10 +7,11 @@ import cats.effect.IO import cats.implicits._ import cats.effect.unsafe.implicits._ import ch.epfl.bluebrain.nexus.delta.kernel.Logger -import ch.epfl.bluebrain.nexus.delta.plugins.archive.FileSelf.ParsingError +import ch.epfl.bluebrain.nexus.delta.plugins.storage.FileSelf.ParsingError import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.ArchiveReference.{FileReference, FileSelfReference, ResourceReference} import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.ArchiveRejection._ import ch.epfl.bluebrain.nexus.delta.plugins.archive.model._ +import ch.epfl.bluebrain.nexus.delta.plugins.storage.FileSelf import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.Files import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileId, FileRejection} import ch.epfl.bluebrain.nexus.delta.rdf.implicits._ diff --git a/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchivePluginModule.scala b/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchivePluginModule.scala index 5b49c31151..ad8e79970d 100644 --- a/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchivePluginModule.scala +++ b/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchivePluginModule.scala @@ -4,6 +4,7 @@ import cats.effect.{Clock, IO} import ch.epfl.bluebrain.nexus.delta.kernel.utils.{ClasspathResourceLoader, UUIDF} import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.contexts import ch.epfl.bluebrain.nexus.delta.plugins.archive.routes.ArchiveRoutes +import ch.epfl.bluebrain.nexus.delta.plugins.storage.FileSelf import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.Files import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution} diff --git a/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/model/ArchiveRejection.scala b/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/model/ArchiveRejection.scala index c35a238a66..7043828166 100644 --- a/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/model/ArchiveRejection.scala +++ b/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/model/ArchiveRejection.scala @@ -3,7 +3,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.archive.model import akka.http.scaladsl.model.StatusCodes import ch.epfl.bluebrain.nexus.delta.kernel.error.Rejection import ch.epfl.bluebrain.nexus.delta.kernel.utils.ClassUtils -import ch.epfl.bluebrain.nexus.delta.plugins.archive.FileSelf +import ch.epfl.bluebrain.nexus.delta.plugins.storage.FileSelf import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.AbsolutePath import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri diff --git a/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownloadSpec.scala b/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownloadSpec.scala index 1a3fc7a729..a48e49a687 100644 --- a/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownloadSpec.scala +++ b/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownloadSpec.scala @@ -9,11 +9,11 @@ import akka.util.ByteString import cats.data.NonEmptySet import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.utils.UrlUtils.encode -import ch.epfl.bluebrain.nexus.delta.plugins.archive.FileSelf.ParsingError +import ch.epfl.bluebrain.nexus.delta.plugins.storage.FileSelf.ParsingError import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.ArchiveReference.{FileReference, FileSelfReference, ResourceReference} import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.ArchiveRejection.{InvalidFileSelf, ResourceNotFound} import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.{ArchiveRejection, ArchiveValue} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.RemoteContextResolutionFixture +import ch.epfl.bluebrain.nexus.delta.plugins.storage.{FileSelf, RemoteContextResolutionFixture} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Client import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection.FileNotFound diff --git a/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveRoutesSpec.scala b/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveRoutesSpec.scala index 9bbc3f8776..b853bf0d58 100644 --- a/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveRoutesSpec.scala +++ b/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveRoutesSpec.scala @@ -11,7 +11,7 @@ import akka.util.ByteString import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.utils.UrlUtils.encode import ch.epfl.bluebrain.nexus.delta.kernel.utils.{StatefulUUIDF, UUIDF} -import ch.epfl.bluebrain.nexus.delta.plugins.archive.FileSelf.ParsingError.InvalidPath +import ch.epfl.bluebrain.nexus.delta.plugins.storage.FileSelf.ParsingError.InvalidPath import ch.epfl.bluebrain.nexus.delta.plugins.archive.routes.ArchiveRoutes import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.ComputedDigest diff --git a/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/FileSelf.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/FileSelf.scala similarity index 97% rename from delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/FileSelf.scala rename to delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/FileSelf.scala index 7a32524bfa..89fb85e57a 100644 --- a/delta/plugins/archive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/FileSelf.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/FileSelf.scala @@ -1,10 +1,10 @@ -package ch.epfl.bluebrain.nexus.delta.plugins.archive +package ch.epfl.bluebrain.nexus.delta.plugins.storage import akka.http.scaladsl.model.Uri import cats.effect.IO import cats.syntax.all._ import ch.epfl.bluebrain.nexus.delta.kernel.utils.UrlUtils -import ch.epfl.bluebrain.nexus.delta.plugins.archive.FileSelf.ParsingError._ +import ch.epfl.bluebrain.nexus.delta.plugins.storage.FileSelf.ParsingError._ import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.implicits._ import ch.epfl.bluebrain.nexus.delta.sdk.error.SDKError diff --git a/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/FileSelfSuite.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileSelfSuite.scala similarity index 94% rename from delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/FileSelfSuite.scala rename to delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileSelfSuite.scala index 69249e687b..1a5661e005 100644 --- a/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/FileSelfSuite.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileSelfSuite.scala @@ -1,7 +1,8 @@ -package ch.epfl.bluebrain.nexus.delta.plugins.archive +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files import ch.epfl.bluebrain.nexus.delta.kernel.utils.UrlUtils.encode -import ch.epfl.bluebrain.nexus.delta.plugins.archive.FileSelf.ParsingError._ +import ch.epfl.bluebrain.nexus.delta.plugins.storage.FileSelf +import ch.epfl.bluebrain.nexus.delta.plugins.storage.FileSelf.ParsingError._ import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv import ch.epfl.bluebrain.nexus.delta.rdf.implicits._ diff --git a/delta/sourcing-psql/src/main/resources/scripts/postgres/drop/drop-tables.ddl b/delta/sourcing-psql/src/main/resources/scripts/postgres/drop/drop-tables.ddl index db79608e49..eac9b7e574 100644 --- a/delta/sourcing-psql/src/main/resources/scripts/postgres/drop/drop-tables.ddl +++ b/delta/sourcing-psql/src/main/resources/scripts/postgres/drop/drop-tables.ddl @@ -1,3 +1,4 @@ +DROP TABLE IF EXISTS public.ship_original_project_context; DROP TABLE IF EXISTS public.ship_reports; DROP TABLE IF EXISTS public.global_events; DROP TABLE IF EXISTS public.global_states; diff --git a/delta/sourcing-psql/src/main/resources/scripts/postgres/init/V1_10_M09_001__ship_original_project_context.ddl b/delta/sourcing-psql/src/main/resources/scripts/postgres/init/V1_10_M09_001__ship_original_project_context.ddl new file mode 100644 index 0000000000..7dd3b1d379 --- /dev/null +++ b/delta/sourcing-psql/src/main/resources/scripts/postgres/init/V1_10_M09_001__ship_original_project_context.ddl @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS public.ship_original_project_context( + org text NOT NULL, + project text NOT NULL, + context JSONB NOT NULL, + PRIMARY KEY(org, project) +) + diff --git a/ship/src/main/resources/ship-default.conf b/ship/src/main/resources/ship-default.conf index 2c6404e7b6..261d435892 100644 --- a/ship/src/main/resources/ship-default.conf +++ b/ship/src/main/resources/ship-default.conf @@ -39,7 +39,8 @@ ship { } input { - base-uri = "http://localhost:8080/v1" + original-base-uri = "http://localhost:8080/v1" + target-base-uri = "http://localhost:8080/v1" event-log { query-config = { diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/ContextWiring.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/ContextWiring.scala index d1a7e7dbdd..2d587441c4 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/ContextWiring.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/ContextWiring.scala @@ -5,8 +5,8 @@ import ch.epfl.bluebrain.nexus.delta.kernel.utils.ClasspathResourceLoader import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.{contexts => bgContexts} import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.{contexts => compositeViewContexts} import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.{contexts => esContexts} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{contexts => storageContext} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{contexts => fileContext} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{contexts => storageContext} import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.contexts import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala index 1324a30e48..18ecd56f98 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala @@ -2,6 +2,7 @@ package ch.epfl.bluebrain.nexus.ship import cats.effect.{Clock, IO} import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF +import ch.epfl.bluebrain.nexus.delta.plugins.storage.FileSelf import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.{JsonLdApi, JsonLdJavaApi} import ch.epfl.bluebrain.nexus.delta.rdf.shacl.ValidateShacl import ch.epfl.bluebrain.nexus.delta.sdk.organizations.FetchActiveOrganization @@ -14,8 +15,9 @@ import ch.epfl.bluebrain.nexus.ship.config.InputConfig import ch.epfl.bluebrain.nexus.ship.organizations.OrganizationProvider import ch.epfl.bluebrain.nexus.ship.projects.ProjectProcessor import ch.epfl.bluebrain.nexus.ship.resolvers.ResolverProcessor -import ch.epfl.bluebrain.nexus.ship.resources.{ResourceProcessor, ResourceWiring} +import ch.epfl.bluebrain.nexus.ship.resources.{DistributionPatcher, ResourceProcessor, ResourceWiring} import ch.epfl.bluebrain.nexus.ship.schemas.{SchemaProcessor, SchemaWiring} +import ch.epfl.bluebrain.nexus.ship.projects.OriginalProjectContext import ch.epfl.bluebrain.nexus.ship.views.{BlazegraphViewProcessor, CompositeViewProcessor, ElasticSearchViewProcessor} import fs2.Stream @@ -28,12 +30,14 @@ object RunShip { implicit val jsonLdApi: JsonLdApi = JsonLdJavaApi.lenient for { report <- { - val orgProvider = + val orgProvider = OrganizationProvider(config.eventLog, config.serviceAccount.value, xas, clock)(uuidF) - val fetchContext = FetchContext(ApiMappings.empty, xas, Quotas.disabled) - val eventLogConfig = config.eventLog - val baseUri = config.baseUri - val projectMapper = ProjectMapper(config.projectMapping) + val fetchContext = FetchContext(ApiMappings.empty, xas, Quotas.disabled) + val originalProjectContext = new OriginalProjectContext(xas) + val eventLogConfig = config.eventLog + val originalBaseUri = config.originalBaseUri + val targetBaseUri = config.targetBaseUri + val projectMapper = ProjectMapper(config.projectMapping) for { // Provision organizations _ <- orgProvider.create(config.organizations.values) @@ -48,10 +52,12 @@ object RunShip { rcr = ContextWiring.resolverContextResolution(fetchResource, fetchContext, remoteContextResolution, eventLogConfig, eventClock, xas) schemaImports = SchemaWiring.schemaImports(fetchResource, fetchSchema, fetchContext, eventLogConfig, eventClock, xas) // Processors - projectProcessor <- ProjectProcessor(fetchActiveOrg, fetchContext, rcr, projectMapper, config, eventClock, xas)(baseUri, jsonLdApi) + projectProcessor <- ProjectProcessor(fetchActiveOrg, fetchContext, rcr, originalProjectContext, projectMapper, config, eventClock, xas)(targetBaseUri, jsonLdApi) resolverProcessor = ResolverProcessor(fetchContext, projectMapper, eventLogConfig, eventClock, xas) schemaProcessor = SchemaProcessor(schemaLog, fetchContext, schemaImports, rcr, projectMapper, eventClock) - resourceProcessor = ResourceProcessor(resourceLog, rcr, projectMapper, fetchContext, eventClock) + fileSelf = FileSelf(originalProjectContext)(originalBaseUri) + distributionPatcher = new DistributionPatcher(fileSelf, projectMapper, targetBaseUri) + resourceProcessor = ResourceProcessor(resourceLog, rcr, projectMapper, fetchContext, distributionPatcher, eventClock) esViewsProcessor = ElasticSearchViewProcessor(fetchContext, rcr, projectMapper, eventLogConfig, eventClock, xas) bgViewsProcessor = BlazegraphViewProcessor(fetchContext, rcr, projectMapper, eventLogConfig, eventClock, xas) compositeViewsProcessor = CompositeViewProcessor(fetchContext, rcr, projectMapper, eventLogConfig, eventClock, xas) diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/InputConfig.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/InputConfig.scala index 96113adf16..c469556291 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/InputConfig.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/InputConfig.scala @@ -15,7 +15,8 @@ import pureconfig.error.{CannotConvert, FailureReason} import pureconfig.generic.semiauto.deriveReader final case class InputConfig( - baseUri: BaseUri, + originalBaseUri: BaseUri, + targetBaseUri: BaseUri, eventLog: EventLogConfig, organizations: OrganizationCreationConfig, projectMapping: ProjectMapping = Map.empty, diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/OriginalProjectContext.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/OriginalProjectContext.scala new file mode 100644 index 0000000000..936d365c4e --- /dev/null +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/OriginalProjectContext.scala @@ -0,0 +1,49 @@ +package ch.epfl.bluebrain.nexus.ship.projects + +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext +import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.{ApiMappings, ProjectContext} +import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors +import ch.epfl.bluebrain.nexus.delta.sourcing.implicits._ +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Identity, ProjectRef} +import doobie.implicits._ +import io.circe.generic.semiauto.deriveCodec +import io.circe.syntax.EncoderOps +import io.circe.{Codec, Json} + +/** + * Allows to keep track of the original project context which will change because of project renaming / patching + * configuration so as to be able to use them to expand iris in resource payload + */ +class OriginalProjectContext(xas: Transactors) extends FetchContext { + + implicit val projectContextCodec: Codec[ProjectContext] = deriveCodec[ProjectContext] + + override def defaultApiMappings: ApiMappings = ApiMappings.empty + + override def onRead(ref: ProjectRef): IO[ProjectContext] = + sql"""|SELECT context + |FROM public.ship_original_project_context + |WHERE org = ${ref.organization} + |AND project = ${ref.project}""".stripMargin.query[Json].unique.transact(xas.read).flatMap { json => + IO.fromEither(json.as[ProjectContext]) + } + + override def onCreate(ref: ProjectRef)(implicit subject: Identity.Subject): IO[ProjectContext] = + IO.raiseError(new IllegalStateException("OnCreate should not be called in this context")) + + override def onModify(ref: ProjectRef)(implicit subject: Identity.Subject): IO[ProjectContext] = + IO.raiseError(new IllegalStateException("OnCreate should not be called in this context")) + + def save(project: ProjectRef, context: ProjectContext): IO[Unit] = + sql"""INSERT INTO public.ship_original_project_context (org, project, context) + |VALUES ( + | ${project.organization}, ${project.project}, ${context.asJson} + |) + |ON CONFLICT (org, project) + |DO UPDATE set + | context = EXCLUDED.context; + |""".stripMargin.update.run + .transact(xas.write) + .void +} diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala index ddceb3c8a6..b46f1d386f 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala @@ -8,7 +8,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.model.BaseUri import ch.epfl.bluebrain.nexus.delta.sdk.organizations.FetchActiveOrganization import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ProjectEvent._ import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ProjectRejection.NotFound -import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.{ApiMappings, ProjectEvent, ProjectFields, ProjectRejection} +import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.{ApiMappings, ProjectBase, ProjectContext, ProjectEvent, ProjectFields, ProjectRejection} import ch.epfl.bluebrain.nexus.delta.sdk.projects.{FetchContext, Projects, ProjectsImpl, ValidateProjectDeletion} import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors @@ -22,6 +22,7 @@ import io.circe.Decoder final class ProjectProcessor private ( projects: Projects, + originalProjectContext: OriginalProjectContext, projectMapper: ProjectMapper, clock: EventClock, uuidF: EventUUIDF, @@ -46,12 +47,16 @@ final class ProjectProcessor private ( event match { case ProjectCreated(_, _, _, _, _, description, apiMappings, base, vocab, enforceSchema, _, _) => - val fields = ProjectFields(description, apiMappings, Some(base), Some(vocab), enforceSchema) - projects.create(projectRef, fields) >> + val fields = ProjectFields(description, apiMappings, Some(base), Some(vocab), enforceSchema) + val context = ProjectContext(apiMappings, ProjectBase.unsafe(base.value), vocab.value, enforceSchema) + originalProjectContext.save(projectRef, context) >> + projects.create(projectRef, fields) >> scopeInitializer.initializeProject(projectRef) case ProjectUpdated(_, _, _, _, _, description, apiMappings, base, vocab, enforceSchema, _, _) => - val fields = ProjectFields(description, apiMappings, Some(base), Some(vocab), enforceSchema) - projects.update(projectRef, cRev, fields) + val fields = ProjectFields(description, apiMappings, Some(base), Some(vocab), enforceSchema) + val context = ProjectContext(apiMappings, ProjectBase.unsafe(base.value), vocab.value, enforceSchema) + originalProjectContext.save(projectRef, context) >> + projects.update(projectRef, cRev, fields) case _: ProjectDeprecated => projects.deprecate(projectRef, cRev) case _: ProjectUndeprecated => @@ -76,6 +81,7 @@ object ProjectProcessor { fetchActiveOrg: FetchActiveOrganization, fetchContext: FetchContext, rcr: ResolverContextResolution, + originalProjectContext: OriginalProjectContext, projectMapper: ProjectMapper, config: InputConfig, clock: EventClock, @@ -98,6 +104,6 @@ object ProjectProcessor { xas, clock )(base, uuidF) - new ProjectProcessor(projects, projectMapper, clock, uuidF, initializer) + new ProjectProcessor(projects, originalProjectContext, projectMapper, clock, uuidF, initializer) } } diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resources/DistributionPatcher.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resources/DistributionPatcher.scala new file mode 100644 index 0000000000..d8a086ada4 --- /dev/null +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resources/DistributionPatcher.scala @@ -0,0 +1,57 @@ +package ch.epfl.bluebrain.nexus.ship.resources + +import cats.effect.IO +import cats.syntax.all._ +import ch.epfl.bluebrain.nexus.delta.kernel.Logger +import ch.epfl.bluebrain.nexus.delta.plugins.storage.FileSelf +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.defaultS3StorageId +import ch.epfl.bluebrain.nexus.delta.rdf.utils.UriUtils +import ch.epfl.bluebrain.nexus.delta.sdk.model.{BaseUri, ResourceUris} +import ch.epfl.bluebrain.nexus.ship.ProjectMapper +import ch.epfl.bluebrain.nexus.ship.resources.DistributionPatcher._ +import io.circe.Json +import io.circe.optics.JsonPath.root +import io.circe.syntax.KeyOps + +final class DistributionPatcher(fileSelfParser: FileSelf, projectMapper: ProjectMapper, targetBase: BaseUri) { + + /** + * Distribution may be defined as an object or as an array in original payloads + */ + def singleOrArray: Json => IO[Json] = root.distribution.json.modifyA { json => + json.asArray match { + case Some(array) => array.traverse(single).map(Json.arr(_: _*)) + case None => single(json) + } + }(_) + + private[resources] def single: Json => IO[Json] = (json: Json) => fileContentUrl(json).map(toS3Location) + + private def toS3Location: Json => Json = root.atLocation.store.json.replace(targetStorage) + + private def fileContentUrl: Json => IO[Json] = root.contentUrl.string.modifyA { string: String => + for { + uri <- parseAsUri(string) + fileSelfAttempt <- fileSelfParser.parse(uri).attempt + result <- fileSelfAttempt match { + case Right((project, resourceRef)) => + val targetProject = projectMapper.map(project) + IO.pure(ResourceUris("files", targetProject, resourceRef.original).accessUri(targetBase).toString()) + case Left(error) => + // We log and keep the value + logger.error(error)(s"'$string' could not be parsed as a file self").as(string) + } + } yield result + }(_) + + private def parseAsUri(string: String) = IO.fromEither(UriUtils.uri(string).leftMap(new IllegalArgumentException(_))) + +} + +object DistributionPatcher { + private val logger = Logger[DistributionPatcher] + + // All files are moved to a storage in S3 with a stable id + private val targetStorage = Json.obj("@id" := defaultS3StorageId, "@type" := "S3Storage", "_rev" := 1) + +} diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resources/ResourceProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resources/ResourceProcessor.scala index 2a6919b67c..4d2c271eeb 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resources/ResourceProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resources/ResourceProcessor.scala @@ -19,8 +19,12 @@ import ch.epfl.bluebrain.nexus.ship.resources.ResourceProcessor.logger import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, ImportStatus, ProjectMapper} import io.circe.Decoder -class ResourceProcessor private (resources: Resources, projectMapper: ProjectMapper, clock: EventClock) - extends EventProcessor[ResourceEvent] { +class ResourceProcessor private ( + resources: Resources, + projectMapper: ProjectMapper, + distributionPatcher: DistributionPatcher, + clock: EventClock +) extends EventProcessor[ResourceEvent] { override def resourceType: EntityType = Resources.entityType @@ -44,9 +48,13 @@ class ResourceProcessor private (resources: Resources, projectMapper: ProjectMap event match { case e: ResourceCreated => - resources.create(e.id, project, e.schema.toIdSegment, e.source, e.tag) + distributionPatcher.singleOrArray(e.source).flatMap { patched => + resources.create(e.id, project, e.schema.toIdSegment, patched, e.tag) + } case e: ResourceUpdated => - resources.update(e.id, project, e.schema.toIdSegment.some, cRev, e.source, e.tag) + distributionPatcher.singleOrArray(e.source).flatMap { patched => + resources.update(e.id, project, e.schema.toIdSegment.some, cRev, patched, e.tag) + } case e: ResourceSchemaUpdated => resources.updateAttachedSchema(e.id, project, e.schema.toIdSegment) case e: ResourceRefreshed => @@ -80,10 +88,11 @@ object ResourceProcessor { rcr: ResolverContextResolution, projectMapper: ProjectMapper, fetchContext: FetchContext, + distributionPatcher: DistributionPatcher, clock: EventClock )(implicit jsonLdApi: JsonLdApi): ResourceProcessor = { val resources = ResourcesImpl(log, fetchContext, rcr) - new ResourceProcessor(resources, projectMapper, clock) + new ResourceProcessor(resources, projectMapper, distributionPatcher, clock) } } diff --git a/ship/src/test/resources/config/external.conf b/ship/src/test/resources/config/external.conf index 6a1757372a..ab56f2e472 100644 --- a/ship/src/test/resources/config/external.conf +++ b/ship/src/test/resources/config/external.conf @@ -1,5 +1,5 @@ ship { input { - base-uri = "https://bbp.epfl.ch/v1" + target-base-uri = "https://bbp.epfl.ch/v1" } } \ No newline at end of file diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigFixtures.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigFixtures.scala index 1589ab58e3..8835112481 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigFixtures.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigFixtures.scala @@ -47,6 +47,7 @@ trait ShipConfigFixtures extends ConfigFixtures with StorageFixtures with Classp def inputConfig: InputConfig = InputConfig( + baseUri, baseUri, eventLogConfig, organizationsCreation, diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigSuite.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigSuite.scala index 1b13254017..66f8f00558 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigSuite.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigSuite.scala @@ -26,7 +26,7 @@ class ShipConfigSuite extends NexusSuite with ShipConfigFixtures with LocalStack test("Default configuration should be parsed and loaded") { val expectedBaseUri = BaseUri("http://localhost:8080", Label.unsafe("v1")) - ShipConfig.load(None).map(_.input.baseUri).assertEquals(expectedBaseUri) + ShipConfig.load(None).map(_.input.targetBaseUri).assertEquals(expectedBaseUri) } test("The defaults (name/description) for views should be correct") { @@ -39,7 +39,7 @@ class ShipConfigSuite extends NexusSuite with ShipConfigFixtures with LocalStack val expectedBaseUri = BaseUri("https://bbp.epfl.ch", Label.unsafe("v1")) for { externalConfigPath <- loader.absolutePath("config/external.conf") - _ <- ShipConfig.load(Some(Path(externalConfigPath))).map(_.input.baseUri).assertEquals(expectedBaseUri) + _ <- ShipConfig.load(Some(Path(externalConfigPath))).map(_.input.targetBaseUri).assertEquals(expectedBaseUri) } yield () } @@ -87,7 +87,7 @@ class ShipConfigSuite extends NexusSuite with ShipConfigFixtures with LocalStack for { _ <- uploadFileToS3(fs2S3client, bucket, configPath) shipConfig <- ShipConfig.loadFromS3(s3Client, bucket, configPath) - _ = assertEquals(shipConfig.input.baseUri.toString, "https://bbp.epfl.ch/v1") + _ = assertEquals(shipConfig.input.targetBaseUri.toString, "https://bbp.epfl.ch/v1") } yield () } diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/projects/OriginalProjectContextSuite.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/projects/OriginalProjectContextSuite.scala new file mode 100644 index 0000000000..1c543ba670 --- /dev/null +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/projects/OriginalProjectContextSuite.scala @@ -0,0 +1,37 @@ +package ch.epfl.bluebrain.nexus.ship.projects + +import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv +import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.{ApiMappings, ProjectBase, ProjectContext} +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef +import ch.epfl.bluebrain.nexus.delta.sourcing.postgres.Doobie +import ch.epfl.bluebrain.nexus.testkit.mu.NexusSuite +import munit.AnyFixture + +class OriginalProjectContextSuite extends NexusSuite with Doobie.Fixture { + + override def munitFixtures: Seq[AnyFixture[_]] = List(doobie) + + private lazy val xas = doobie() + + private lazy val originalProjectContext = new OriginalProjectContext(xas) + + private val project = ProjectRef.unsafe("org", "proj") + + private val context = ProjectContext( + ApiMappings("test" -> (nxv + "test")), + ProjectBase.unsafe(nxv + "base"), + nxv + "vocab", + enforceSchema = true + ) + + test("Save and get back and overwrite") { + for { + _ <- originalProjectContext.save(project, context) + _ <- originalProjectContext.onRead(project).assertEquals(context) + updatedContext = context.copy(enforceSchema = false) + _ <- originalProjectContext.save(project, updatedContext) + _ <- originalProjectContext.onRead(project).assertEquals(updatedContext) + } yield () + } + +} diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/resources/DistributionPatcherSuite.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/resources/DistributionPatcherSuite.scala new file mode 100644 index 0000000000..d39666ede5 --- /dev/null +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/resources/DistributionPatcherSuite.scala @@ -0,0 +1,111 @@ +package ch.epfl.bluebrain.nexus.ship.resources + +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.FileSelf +import ch.epfl.bluebrain.nexus.delta.plugins.storage.FileSelf.ParsingError.InvalidFileId +import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode +import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri +import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv +import ch.epfl.bluebrain.nexus.delta.sdk.model.{BaseUri, ResourceUris} +import ch.epfl.bluebrain.nexus.delta.sdk.syntax._ +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Label, ProjectRef, ResourceRef} +import ch.epfl.bluebrain.nexus.ship.ProjectMapper +import ch.epfl.bluebrain.nexus.testkit.mu.NexusSuite + +class DistributionPatcherSuite extends NexusSuite { + + private val project1 = ProjectRef.unsafe("bbp", "proj1") + private val targetProject1 = ProjectRef.unsafe("obp", "proj1") + + private val resourceIri: Iri = nxv + "resourceId" + + private val prefix = Label.unsafe("v1") + private val originalBaseUri = BaseUri(uri"http://bbp.epfl.ch/nexus", prefix) + private val targetBaseUri = BaseUri(uri"https://www.openbrainplatform.org/api/nexus", prefix) + + private val validFileSelfUri = buildFileSelfUri(project1, resourceIri).accessUri(originalBaseUri) + + private def buildFileSelfUri(project: ProjectRef, id: Iri) = + ResourceUris("files", project, id) + + private val fileSelf = new FileSelf { + override def parse(input: IriOrBNode.Iri): IO[(ProjectRef, ResourceRef)] = + input match { + case value if value.startsWith(originalBaseUri.iriEndpoint) => + IO.pure(project1 -> ResourceRef.Latest(resourceIri)) + case value => IO.raiseError(InvalidFileId(value)) + } + } + + private val patcherNoProjectMapping = new DistributionPatcher(fileSelf, ProjectMapper(Map.empty), targetBaseUri) + private val patcherWithProjectMapping = + new DistributionPatcher(fileSelf, ProjectMapper(Map(project1 -> targetProject1)), targetBaseUri) + + test("Do nothing on a distribution payload without fields to patch") { + val input = json"""{ "anotherField": "XXX" }""" + patcherNoProjectMapping.single(input).assertEquals(input) + } + + test("Patch location on a distribution to point to the new unique S3 storage") { + val input = + json"""{ + "atLocation": { + "store": { + "@id": "https://bbp.epfl.ch/remote-disk-storage", + "@type": "RemoteDiskStorage", + "_rev": 3 + } + } + }""" + val expected = + json"""{ + "atLocation": { + "store": { + "@id": "https://bluebrain.github.io/nexus/vocabulary/defaultS3Storage", + "@type": "S3Storage", + "_rev": 1 + } + } + }""" + patcherWithProjectMapping.single(input).assertEquals(expected) + } + + test("Patching an invalid file self should preserve the initial value") { + val input = json"""{ "contentUrl": "xxx" }""" + patcherNoProjectMapping.single(input).assertEquals(input) + } + + test("Patch a valid file self on a distribution without project mapping") { + val input = json"""{ "contentUrl": "$validFileSelfUri" }""" + val expectedContentUri = buildFileSelfUri(project1, resourceIri).accessUri(targetBaseUri) + val expected = json"""{ "contentUrl": "$expectedContentUri" }""" + patcherNoProjectMapping.single(input).assertEquals(expected) + } + + test("Patch a valid file self on a distribution with project mapping") { + val input = json"""{ "contentUrl": "$validFileSelfUri" }""" + val expectedContentUri = buildFileSelfUri(targetProject1, resourceIri).accessUri(targetBaseUri) + val expected = json"""{ "contentUrl": "$expectedContentUri" }""" + patcherWithProjectMapping.single(input).assertEquals(expected) + } + + test("Patch an invalid distribution self should preserve the initial value") { + val input = json"""{ "distribution":"xxx" }""" + patcherNoProjectMapping.singleOrArray(input).assertEquals(input) + } + + test("Patch a valid file self on a distribution as an object") { + val input = json"""{ "distribution": { "contentUrl": "$validFileSelfUri" } }""" + val expectedContentUri = buildFileSelfUri(project1, resourceIri).accessUri(targetBaseUri) + val expected = json"""{ "distribution": { "contentUrl": "$expectedContentUri" } }""" + patcherNoProjectMapping.singleOrArray(input).assertEquals(expected) + } + + test("Patch a valid file self on a distribution as an array") { + val input = json"""{ "distribution": [{ "contentUrl": "$validFileSelfUri" }] }""" + val expectedContentUri = buildFileSelfUri(project1, resourceIri).accessUri(targetBaseUri) + val expected = json"""{ "distribution": [{ "contentUrl": "$expectedContentUri" }] }""" + patcherNoProjectMapping.singleOrArray(input).assertEquals(expected) + } + +}