From 9c9fefe36c9e9331f75a670fe011e06055703ca8 Mon Sep 17 00:00:00 2001 From: Oliver <20188437+olivergrabinski@users.noreply.github.com> Date: Tue, 2 Apr 2024 10:58:17 +0200 Subject: [PATCH 1/7] Add project mapping to ship config --- .../plugins/search/model/SearchConfig.scala | 11 ----------- .../nexus/delta/sourcing/model/ProjectRef.scala | 13 +++++++++++++ .../bluebrain/nexus/ship/config/ShipConfig.scala | 16 +++++++++++++++- .../nexus/ship/projects/ProjectProcessor.scala | 7 ++++--- .../test/resources/config/project-mapping.conf | 5 +++++ .../ch/epfl/bluebrain/nexus/ship/MainSuite.scala | 11 +++++++++++ .../epfl/bluebrain/nexus/ship/RunShipSuite.scala | 4 ---- .../nexus/ship/config/ShipConfigSuite.scala | 15 ++++++++++++++- 8 files changed, 62 insertions(+), 20 deletions(-) create mode 100644 ship/src/test/resources/config/project-mapping.conf create mode 100644 ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/MainSuite.scala diff --git a/delta/plugins/search/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/search/model/SearchConfig.scala b/delta/plugins/search/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/search/model/SearchConfig.scala index 83c8ed361b..7b862445a0 100644 --- a/delta/plugins/search/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/search/model/SearchConfig.scala +++ b/delta/plugins/search/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/search/model/SearchConfig.scala @@ -34,17 +34,6 @@ final case class SearchConfig( object SearchConfig { - implicit val projectRefReader: ConfigReader[ProjectRef] = ConfigReader.fromString { value => - value.split("/").toList match { - case orgStr :: projectStr :: Nil => - (Label(orgStr), Label(projectStr)) - .mapN(ProjectRef(_, _)) - .leftMap(err => CannotConvert(value, classOf[ProjectRef].getSimpleName, err.getMessage)) - case _ => - Left(CannotConvert(value, classOf[ProjectRef].getSimpleName, "Wrong format")) - } - } - type Suite = Set[ProjectRef] type Suites = Map[Label, Suite] diff --git a/delta/sourcing-psql/src/main/scala/ch/epfl/bluebrain/nexus/delta/sourcing/model/ProjectRef.scala b/delta/sourcing-psql/src/main/scala/ch/epfl/bluebrain/nexus/delta/sourcing/model/ProjectRef.scala index 5e2097d358..6292dda70b 100644 --- a/delta/sourcing-psql/src/main/scala/ch/epfl/bluebrain/nexus/delta/sourcing/model/ProjectRef.scala +++ b/delta/sourcing-psql/src/main/scala/ch/epfl/bluebrain/nexus/delta/sourcing/model/ProjectRef.scala @@ -9,6 +9,8 @@ import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.decoder.JsonLdDecoderError.Parsi import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.encoder.JsonLdEncoder import doobie.{Get, Put} import io.circe.{Decoder, Encoder, KeyDecoder, KeyEncoder} +import pureconfig.ConfigReader +import pureconfig.error.CannotConvert /** * A project label along with its parent organization label. @@ -70,4 +72,15 @@ object ProjectRef { (projectRef.organization.value, projectRef.project.value) } + implicit val projectRefConfigReader: ConfigReader[ProjectRef] = ConfigReader.fromString { value => + value.split("/").toList match { + case orgStr :: projectStr :: Nil => + (Label(orgStr), Label(projectStr)) + .mapN(ProjectRef(_, _)) + .leftMap(err => CannotConvert(value, classOf[ProjectRef].getSimpleName, err.getMessage)) + case _ => + Left(CannotConvert(value, classOf[ProjectRef].getSimpleName, "Wrong format")) + } + } + } diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfig.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfig.scala index e83228e609..7107a279ed 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfig.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfig.scala @@ -1,12 +1,17 @@ package ch.epfl.bluebrain.nexus.ship.config import cats.effect.IO +import cats.implicits._ import ch.epfl.bluebrain.nexus.delta.kernel.config.Configs import ch.epfl.bluebrain.nexus.delta.sdk.model.{BaseUri, ServiceAccountConfig} import ch.epfl.bluebrain.nexus.delta.sourcing.config.{DatabaseConfig, EventLogConfig} +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef +import ch.epfl.bluebrain.nexus.ship.config.ShipConfig.ProjectMapping import com.typesafe.config.Config import fs2.io.file.Path import pureconfig.ConfigReader +import pureconfig.configurable.genericMapReader +import pureconfig.error.CannotConvert import pureconfig.generic.semiauto.deriveReader final case class ShipConfig( @@ -14,13 +19,22 @@ final case class ShipConfig( database: DatabaseConfig, eventLog: EventLogConfig, organizations: OrganizationCreationConfig, + projectMapping: Option[ProjectMapping], serviceAccount: ServiceAccountConfig ) object ShipConfig { - implicit final val shipConfigReader: ConfigReader[ShipConfig] = + type ProjectMapping = Map[ProjectRef, ProjectRef] + + implicit val mapReader: ConfigReader[ProjectMapping] = + genericMapReader(str => + ProjectRef.parse(str).leftMap(e => CannotConvert(str, classOf[ProjectRef].getSimpleName, e)) + ) + + implicit final val shipConfigReader: ConfigReader[ShipConfig] = { deriveReader[ShipConfig] + } def merge(externalConfigPath: Option[Path]): IO[(ShipConfig, Config)] = for { diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala index f149590cfa..92265ccb25 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala @@ -33,9 +33,10 @@ final class ProjectProcessor private (projects: Projects, clock: EventClock, uui } private def evaluateInternal(event: ProjectEvent): IO[ImportStatus] = { - implicit val s: Subject = event.subject - val projectRef = event.project - val cRev = event.rev - 1 + implicit val s: Subject = event.subject + val projectRef = event.project + val cRev = event.rev - 1 + event match { case ProjectCreated(_, _, _, _, _, description, apiMappings, base, vocab, enforceSchema, _, _) => val fields = ProjectFields(description, apiMappings, Some(base), Some(vocab), enforceSchema) diff --git a/ship/src/test/resources/config/project-mapping.conf b/ship/src/test/resources/config/project-mapping.conf new file mode 100644 index 0000000000..bf33645629 --- /dev/null +++ b/ship/src/test/resources/config/project-mapping.conf @@ -0,0 +1,5 @@ +ship { + project-mapping = { + "private/mmb": "obp/reference" + } +} \ No newline at end of file diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/MainSuite.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/MainSuite.scala new file mode 100644 index 0000000000..ce93ff89e5 --- /dev/null +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/MainSuite.scala @@ -0,0 +1,11 @@ +package ch.epfl.bluebrain.nexus.ship + +import ch.epfl.bluebrain.nexus.testkit.mu.NexusSuite + +class MainSuite extends NexusSuite { + + test("Show config") { + Main.showConfig(None) + } + +} diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala index 08593cc71f..4e61cdd909 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala @@ -61,10 +61,6 @@ class RunShipSuite extends NexusSuite with RunShipSuite.Fixture { } yield () } - test("Show config") { - Main.showConfig(None) - } - private def asPath(path: String): IO[Path] = { ClasspathResourceLoader().absolutePath(path).map(Path(_)) } diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigSuite.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigSuite.scala index b5b91a248b..7454e8e390 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigSuite.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigSuite.scala @@ -1,7 +1,7 @@ package ch.epfl.bluebrain.nexus.ship.config import ch.epfl.bluebrain.nexus.delta.sdk.model.BaseUri -import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Label, ProjectRef} import ch.epfl.bluebrain.nexus.testkit.mu.NexusSuite import fs2.io.file.Path @@ -19,4 +19,17 @@ class ShipConfigSuite extends NexusSuite { _ <- ShipConfig.load(Some(Path(externalConfigPath))).map(_.baseUri).assertEquals(expectedBaseUri) } yield () } + + test("Should have correct project mapping") { + val privateMmb = ProjectRef.unsafe("private", "mmb") + val obpReference = ProjectRef.unsafe("obp", "reference") + val expected = Map(privateMmb -> obpReference) + + for { + externalConfigPath <- loader.absolutePath("config/project-mapping.conf") + mapping = ShipConfig.load(Some(Path(externalConfigPath))).map(_.projectMapping) + _ <- mapping.assertEquals(Some(expected)) + } yield () + } + } From 0ca2d0deb09208fd4db0bebe9e82b981bbe70a80 Mon Sep 17 00:00:00 2001 From: Oliver <20188437+olivergrabinski@users.noreply.github.com> Date: Tue, 2 Apr 2024 15:43:13 +0200 Subject: [PATCH 2/7] Add ProjectMapper --- .../bluebrain/nexus/ship/ProjectMapper.scala | 19 ++++++++++++ .../epfl/bluebrain/nexus/ship/RunShip.scala | 29 ++++++++----------- .../ship/projects/ProjectProcessor.scala | 28 ++++++++++++------ .../ship/resolvers/ResolverProcessor.scala | 10 ++++--- .../ship/resources/ResourceProcessor.scala | 25 +++++++++------- .../nexus/ship/schemas/SchemaProcessor.scala | 23 ++++++++------- .../ship/views/BlazegraphViewProcessor.scala | 21 +++++++++----- .../ship/views/CompositeViewProcessor.scala | 17 ++++++----- .../views/ElasticSearchViewProcessor.scala | 21 +++++++++----- .../resources/config/project-mapping.conf | 2 +- 10 files changed, 119 insertions(+), 76 deletions(-) create mode 100644 ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/ProjectMapper.scala diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/ProjectMapper.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/ProjectMapper.scala new file mode 100644 index 0000000000..cfd5c52b3b --- /dev/null +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/ProjectMapper.scala @@ -0,0 +1,19 @@ +package ch.epfl.bluebrain.nexus.ship + +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef +import ch.epfl.bluebrain.nexus.ship.config.ShipConfig.ProjectMapping + +trait ProjectMapper { + def map(project: ProjectRef): ProjectRef +} + +object ProjectMapper { + + def apply(projectMapping: Option[ProjectMapping]): ProjectMapper = + (project: ProjectRef) => + projectMapping match { + case Some(mapping) => mapping.getOrElse(project, project) + case None => project + } + +} diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala index 9f8faefca6..eff2318ee2 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala @@ -40,6 +40,7 @@ class RunShip { val fetchContext = FetchContext(ApiMappings.empty, xas, Quotas.disabled) val eventLogConfig = config.eventLog val baseUri = config.baseUri + val projectMapper = ProjectMapper(config.projectMapping) for { // Provision organizations _ <- orgProvider.create(config.organizations.values) @@ -50,24 +51,18 @@ class RunShip { (schemaLog, fetchSchema) <- SchemaWiring(config.eventLog, eventClock, xas, jsonLdApi) (resourceLog, fetchResource) = ResourceWiring(fetchContext, fetchSchema, eventLogConfig, eventClock, xas) - rcr <- ContextWiring - .resolverContextResolution(fetchResource, fetchContext, eventLogConfig, eventClock, xas) - schemaImports = SchemaWiring.schemaImports( - fetchResource, - fetchSchema, - fetchContext, - eventLogConfig, - eventClock, - xas - ) + // format: off + rcr <- ContextWiring.resolverContextResolution(fetchResource, fetchContext, eventLogConfig, eventClock, xas) + schemaImports = SchemaWiring.schemaImports(fetchResource, fetchSchema, fetchContext, eventLogConfig, eventClock, xas) // Processors - projectProcessor <- ProjectProcessor(fetchActiveOrg, eventLogConfig, eventClock, xas)(baseUri) - resolverProcessor = ResolverProcessor(fetchContext, eventLogConfig, eventClock, xas) - schemaProcessor = SchemaProcessor(schemaLog, fetchContext, schemaImports, rcr, eventClock) - resourceProcessor = ResourceProcessor(resourceLog, fetchContext, eventClock) - esViewsProcessor <- ElasticSearchViewProcessor(fetchContext, rcr, eventLogConfig, eventClock, xas) - bgViewsProcessor = BlazegraphViewProcessor(fetchContext, rcr, eventLogConfig, eventClock, xas) - compositeViewsProcessor = CompositeViewProcessor(fetchContext, rcr, eventLogConfig, eventClock, xas) + projectProcessor <- ProjectProcessor(fetchActiveOrg, projectMapper, eventLogConfig, eventClock, xas)(baseUri) + resolverProcessor = ResolverProcessor(fetchContext, projectMapper, eventLogConfig, eventClock, xas) + schemaProcessor = SchemaProcessor(schemaLog, fetchContext, schemaImports, rcr, projectMapper, eventClock) + resourceProcessor = ResourceProcessor(resourceLog, projectMapper, fetchContext, eventClock) + esViewsProcessor <- ElasticSearchViewProcessor(fetchContext, rcr, projectMapper, eventLogConfig, eventClock, xas) + bgViewsProcessor = BlazegraphViewProcessor(fetchContext, rcr, projectMapper, eventLogConfig, eventClock, xas) + compositeViewsProcessor = CompositeViewProcessor(fetchContext, rcr, projectMapper, eventLogConfig, eventClock, xas) + // format: on report <- EventProcessor .run( events, diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala index 92265ccb25..4a6f5ce81a 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala @@ -15,11 +15,15 @@ import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject import ch.epfl.bluebrain.nexus.delta.sourcing.model.{EntityType, ProjectRef} import ch.epfl.bluebrain.nexus.ship.error.ShipError.ProjectDeletionIsNotAllowed import ch.epfl.bluebrain.nexus.ship.projects.ProjectProcessor.logger -import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, EventUUIDF, ImportStatus} +import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, EventUUIDF, ImportStatus, ProjectMapper} import io.circe.Decoder -final class ProjectProcessor private (projects: Projects, clock: EventClock, uuidF: EventUUIDF) - extends EventProcessor[ProjectEvent] { +final class ProjectProcessor private ( + projects: Projects, + projectMapper: ProjectMapper, + clock: EventClock, + uuidF: EventUUIDF +) extends EventProcessor[ProjectEvent] { override def resourceType: EntityType = Projects.entityType override def decoder: Decoder[ProjectEvent] = ProjectEvent.serializer.codec @@ -33,9 +37,9 @@ final class ProjectProcessor private (projects: Projects, clock: EventClock, uui } private def evaluateInternal(event: ProjectEvent): IO[ImportStatus] = { - implicit val s: Subject = event.subject - val projectRef = event.project - val cRev = event.rev - 1 + implicit val s: Subject = event.subject + val projectRef = projectMapper.map(event.project) + val cRev = event.rev - 1 event match { case ProjectCreated(_, _, _, _, _, description, apiMappings, base, vocab, enforceSchema, _, _) => @@ -64,8 +68,14 @@ final class ProjectProcessor private (projects: Projects, clock: EventClock, uui object ProjectProcessor { private val logger = Logger[ProjectProcessor] - def apply(fetchActiveOrg: FetchActiveOrganization, config: EventLogConfig, clock: EventClock, xas: Transactors)( - implicit base: BaseUri + def apply( + fetchActiveOrg: FetchActiveOrganization, + projectMapper: ProjectMapper, + config: EventLogConfig, + clock: EventClock, + xas: Transactors + )(implicit + base: BaseUri ): IO[ProjectProcessor] = for { uuidF <- EventUUIDF.init() @@ -80,6 +90,6 @@ object ProjectProcessor { xas, clock )(base, uuidF) - new ProjectProcessor(projects, clock, uuidF) + new ProjectProcessor(projects, projectMapper, clock, uuidF) } } diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resolvers/ResolverProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resolvers/ResolverProcessor.scala index 41873888d5..9e3a7d83ae 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resolvers/ResolverProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resolvers/ResolverProcessor.scala @@ -16,10 +16,11 @@ import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject import ch.epfl.bluebrain.nexus.delta.sourcing.model.{EntityType, Identity} import ch.epfl.bluebrain.nexus.ship.resolvers.ResolverProcessor.logger -import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, ImportStatus} +import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, ImportStatus, ProjectMapper} import io.circe.Decoder -class ResolverProcessor private (resolvers: Resolvers, clock: EventClock) extends EventProcessor[ResolverEvent] { +class ResolverProcessor private (resolvers: Resolvers, projectMapper: ProjectMapper, clock: EventClock) + extends EventProcessor[ResolverEvent] { override def resourceType: EntityType = Resolvers.entityType override def decoder: Decoder[ResolverEvent] = ResolverEvent.serializer.codec @@ -34,7 +35,7 @@ class ResolverProcessor private (resolvers: Resolvers, clock: EventClock) extend private def evaluateInternal(event: ResolverEvent): IO[ImportStatus] = { val id = event.id implicit val s: Subject = event.subject - val projectRef = event.project + val projectRef = projectMapper.map(event.project) val cRev = event.rev - 1 event match { case ResolverCreated(_, _, value, _, _, _, _) => @@ -75,11 +76,12 @@ object ResolverProcessor { def apply( fetchContext: FetchContext, + projectMapper: ProjectMapper, config: EventLogConfig, clock: EventClock, xas: Transactors )(implicit api: JsonLdApi): ResolverProcessor = { val resolvers = ResolverWiring.resolvers(fetchContext, config, clock, xas) - new ResolverProcessor(resolvers, clock) + new ResolverProcessor(resolvers, projectMapper, clock) } } diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resources/ResourceProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resources/ResourceProcessor.scala index 9021a4cabf..ec4c46e22d 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resources/ResourceProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resources/ResourceProcessor.scala @@ -16,10 +16,11 @@ import ch.epfl.bluebrain.nexus.delta.sdk.resources.model.ResourceRejection.{Inco import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject import ch.epfl.bluebrain.nexus.delta.sourcing.model.{EntityType, ResourceRef} import ch.epfl.bluebrain.nexus.ship.resources.ResourceProcessor.logger -import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, ImportStatus} +import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, ImportStatus, ProjectMapper} import io.circe.Decoder -class ResourceProcessor private (resources: Resources, clock: EventClock) extends EventProcessor[ResourceEvent] { +class ResourceProcessor private (resources: Resources, projectMapper: ProjectMapper, clock: EventClock) + extends EventProcessor[ResourceEvent] { override def resourceType: EntityType = Resources.entityType @@ -35,6 +36,7 @@ class ResourceProcessor private (resources: Resources, clock: EventClock) extend implicit val s: Subject = event.subject implicit val c: Caller = Caller(s, Set.empty) val cRev = event.rev - 1 + val project = projectMapper.map(event.project) implicit class ResourceRefOps(ref: ResourceRef) { def toIdSegment: IdSegment = IdSegmentRef(ref).value @@ -42,21 +44,21 @@ class ResourceProcessor private (resources: Resources, clock: EventClock) extend event match { case e: ResourceCreated => - resources.create(e.id, e.project, e.schema.toIdSegment, e.source, e.tag) + resources.create(e.id, project, e.schema.toIdSegment, e.source, e.tag) case e: ResourceUpdated => - resources.update(e.id, event.project, e.schema.toIdSegment.some, cRev, e.source, e.tag) + resources.update(e.id, project, e.schema.toIdSegment.some, cRev, e.source, e.tag) case e: ResourceSchemaUpdated => - resources.updateAttachedSchema(e.id, e.project, e.schema.toIdSegment) + resources.updateAttachedSchema(e.id, project, e.schema.toIdSegment) case e: ResourceRefreshed => - resources.refresh(e.id, e.project, e.schema.toIdSegment.some) + resources.refresh(e.id, project, e.schema.toIdSegment.some) case e: ResourceTagAdded => - resources.tag(e.id, e.project, None, e.tag, e.targetRev, cRev) + resources.tag(e.id, project, None, e.tag, e.targetRev, cRev) case e: ResourceTagDeleted => - resources.deleteTag(e.id, e.project, None, e.tag, cRev) + resources.deleteTag(e.id, project, None, e.tag, cRev) case e: ResourceDeprecated => - resources.deprecate(e.id, e.project, None, cRev) + resources.deprecate(e.id, project, None, cRev) case e: ResourceUndeprecated => - resources.undeprecate(e.id, e.project, None, cRev) + resources.undeprecate(e.id, project, None, cRev) } }.redeemWith( { @@ -75,12 +77,13 @@ object ResourceProcessor { def apply( log: ResourceLog, + projectMapper: ProjectMapper, fetchContext: FetchContext, clock: EventClock )(implicit jsonLdApi: JsonLdApi): ResourceProcessor = { val rcr = ResolverContextResolution.never // TODO: Pass correct ResolverContextResolution val resources = ResourcesImpl(log, fetchContext, rcr) - new ResourceProcessor(resources, clock) + new ResourceProcessor(resources, projectMapper, clock) } } diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/schemas/SchemaProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/schemas/SchemaProcessor.scala index 74e9ebf395..cad982f875 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/schemas/SchemaProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/schemas/SchemaProcessor.scala @@ -14,10 +14,11 @@ import ch.epfl.bluebrain.nexus.delta.sdk.schemas.{SchemaImports, Schemas, Schema import ch.epfl.bluebrain.nexus.delta.sourcing.model.EntityType import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject import ch.epfl.bluebrain.nexus.ship.schemas.SchemaProcessor.logger -import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, FailingUUID, ImportStatus} +import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, FailingUUID, ImportStatus, ProjectMapper} import io.circe.Decoder -class SchemaProcessor private (schemas: Schemas, clock: EventClock) extends EventProcessor[SchemaEvent] { +class SchemaProcessor private (schemas: Schemas, projectMapper: ProjectMapper, clock: EventClock) + extends EventProcessor[SchemaEvent] { override def resourceType: EntityType = Schemas.entityType @@ -34,15 +35,16 @@ class SchemaProcessor private (schemas: Schemas, clock: EventClock) extends Even implicit val s: Subject = event.subject implicit val c: Caller = Caller(s, Set.empty) val cRev = event.rev - 1 + val project = projectMapper.map(event.project) event match { - case e: SchemaCreated => schemas.create(e.id, e.project, e.source) - case e: SchemaUpdated => schemas.update(e.id, e.project, cRev, e.source) - case e: SchemaRefreshed => schemas.refresh(e.id, e.project) - case e: SchemaTagAdded => schemas.tag(e.id, e.project, e.tag, e.targetRev, cRev) - case e: SchemaTagDeleted => schemas.deleteTag(e.id, e.project, e.tag, cRev) - case e: SchemaDeprecated => schemas.deprecate(e.id, e.project, cRev) - case e: SchemaUndeprecated => schemas.undeprecate(e.id, e.project, cRev) + case e: SchemaCreated => schemas.create(e.id, project, e.source) + case e: SchemaUpdated => schemas.update(e.id, project, cRev, e.source) + case e: SchemaRefreshed => schemas.refresh(e.id, project) + case e: SchemaTagAdded => schemas.tag(e.id, project, e.tag, e.targetRev, cRev) + case e: SchemaTagDeleted => schemas.deleteTag(e.id, project, e.tag, cRev) + case e: SchemaDeprecated => schemas.deprecate(e.id, project, cRev) + case e: SchemaUndeprecated => schemas.undeprecate(e.id, project, cRev) } }.redeemWith( { @@ -64,10 +66,11 @@ object SchemaProcessor { fetchContext: FetchContext, schemaImports: SchemaImports, rcr: ResolverContextResolution, + projectMapper: ProjectMapper, clock: EventClock )(implicit jsonLdApi: JsonLdApi): SchemaProcessor = { val schemas = SchemasImpl(log, fetchContext, schemaImports, rcr)(jsonLdApi, FailingUUID) - new SchemaProcessor(schemas, clock) + new SchemaProcessor(schemas, projectMapper, clock) } } diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/BlazegraphViewProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/BlazegraphViewProcessor.scala index 4714aba7fa..fea2c60241 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/BlazegraphViewProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/BlazegraphViewProcessor.scala @@ -15,13 +15,16 @@ import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig import ch.epfl.bluebrain.nexus.delta.sourcing.model.EntityType import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject import ch.epfl.bluebrain.nexus.ship.views.BlazegraphViewProcessor.logger -import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, ImportStatus} +import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, ImportStatus, ProjectMapper} import io.circe.Decoder import java.util.UUID -class BlazegraphViewProcessor private (views: UUID => IO[BlazegraphViews], clock: EventClock) - extends EventProcessor[BlazegraphViewEvent] { +class BlazegraphViewProcessor private ( + views: UUID => IO[BlazegraphViews], + projectMapper: ProjectMapper, + clock: EventClock +) extends EventProcessor[BlazegraphViewEvent] { override def resourceType: EntityType = BlazegraphViews.entityType @@ -36,11 +39,12 @@ class BlazegraphViewProcessor private (views: UUID => IO[BlazegraphViews], clock private def evaluateInternal(event: BlazegraphViewEvent): IO[ImportStatus] = { implicit val s: Subject = event.subject val cRev = event.rev - 1 + val project = projectMapper.map(event.project) event match { - case e: BlazegraphViewCreated => views(event.uuid).flatMap(_.create(e.id, e.project, e.value)) - case e: BlazegraphViewUpdated => views(event.uuid).flatMap(_.update(e.id, e.project, cRev, e.value)) - case e: BlazegraphViewDeprecated => views(event.uuid).flatMap(_.deprecate(e.id, e.project, cRev)) - case e: BlazegraphViewUndeprecated => views(event.uuid).flatMap(_.undeprecate(e.id, e.project, cRev)) + case e: BlazegraphViewCreated => views(event.uuid).flatMap(_.create(e.id, project, e.value)) + case e: BlazegraphViewUpdated => views(event.uuid).flatMap(_.update(e.id, project, cRev, e.value)) + case e: BlazegraphViewDeprecated => views(event.uuid).flatMap(_.deprecate(e.id, project, cRev)) + case e: BlazegraphViewUndeprecated => views(event.uuid).flatMap(_.undeprecate(e.id, project, cRev)) case _: BlazegraphViewTagAdded => IO.unit // TODO: Can we tag? } }.redeemWith( @@ -60,6 +64,7 @@ object BlazegraphViewProcessor { def apply( fetchContext: FetchContext, rcr: ResolverContextResolution, + projectMapper: ProjectMapper, config: EventLogConfig, clock: EventClock, xas: Transactors @@ -82,7 +87,7 @@ object BlazegraphViewProcessor { xas, clock )(jsonLdApi, UUIDF.fixed(uuid)) - new BlazegraphViewProcessor(views, clock) + new BlazegraphViewProcessor(views, projectMapper, clock) } } diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/CompositeViewProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/CompositeViewProcessor.scala index b17e2223d9..6d0714a09c 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/CompositeViewProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/CompositeViewProcessor.scala @@ -16,13 +16,13 @@ import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig import ch.epfl.bluebrain.nexus.delta.sourcing.model.EntityType import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject import ch.epfl.bluebrain.nexus.ship.views.CompositeViewProcessor.logger -import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, ImportStatus} +import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, ImportStatus, ProjectMapper} import io.circe.Decoder import java.util.UUID import scala.concurrent.duration.DurationInt -class CompositeViewProcessor(views: UUID => IO[CompositeViews], clock: EventClock) +class CompositeViewProcessor(views: UUID => IO[CompositeViews], projectMapper: ProjectMapper, clock: EventClock) extends EventProcessor[CompositeViewEvent] { override def resourceType: EntityType = CompositeViews.entityType @@ -38,12 +38,12 @@ class CompositeViewProcessor(views: UUID => IO[CompositeViews], clock: EventCloc implicit val s: Subject = event.subject implicit val c: Caller = Caller(s, Set.empty) val cRev = event.rev - 1 - + val project = projectMapper.map(event.project) event match { - case e: CompositeViewCreated => views(event.uuid).flatMap(_.create(e.project, e.source)) - case e: CompositeViewUpdated => views(event.uuid).flatMap(_.update(e.id, e.project, cRev, e.source)) - case e: CompositeViewDeprecated => views(event.uuid).flatMap(_.deprecate(e.id, e.project, cRev)) - case e: CompositeViewUndeprecated => views(event.uuid).flatMap(_.undeprecate(e.id, e.project, cRev)) + case e: CompositeViewCreated => views(event.uuid).flatMap(_.create(project, e.source)) + case e: CompositeViewUpdated => views(event.uuid).flatMap(_.update(e.id, project, cRev, e.source)) + case e: CompositeViewDeprecated => views(event.uuid).flatMap(_.deprecate(e.id, project, cRev)) + case e: CompositeViewUndeprecated => views(event.uuid).flatMap(_.undeprecate(e.id, project, cRev)) case _: CompositeViewTagAdded => IO.unit // TODO: Can/should we tag? } }.redeemWith( @@ -63,6 +63,7 @@ object CompositeViewProcessor { def apply( fetchContext: FetchContext, rcr: ResolverContextResolution, + projectMapper: ProjectMapper, config: EventLogConfig, clock: EventClock, xas: Transactors @@ -84,7 +85,7 @@ object CompositeViewProcessor { clock )(jsonLdApi, UUIDF.fixed(uuid)) - new CompositeViewProcessor(views, clock) + new CompositeViewProcessor(views, projectMapper, clock) } diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ElasticSearchViewProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ElasticSearchViewProcessor.scala index ab83d060d0..49433b44bd 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ElasticSearchViewProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ElasticSearchViewProcessor.scala @@ -16,13 +16,16 @@ import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig import ch.epfl.bluebrain.nexus.delta.sourcing.model.EntityType import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject import ch.epfl.bluebrain.nexus.ship.views.ElasticSearchViewProcessor.logger -import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, ImportStatus} +import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, ImportStatus, ProjectMapper} import io.circe.Decoder import java.util.UUID -class ElasticSearchViewProcessor private (views: UUID => IO[ElasticSearchViews], clock: EventClock) - extends EventProcessor[ElasticSearchViewEvent] { +class ElasticSearchViewProcessor private ( + views: UUID => IO[ElasticSearchViews], + projectMapper: ProjectMapper, + clock: EventClock +) extends EventProcessor[ElasticSearchViewEvent] { override def resourceType: EntityType = ElasticSearchViews.entityType @@ -37,11 +40,12 @@ class ElasticSearchViewProcessor private (views: UUID => IO[ElasticSearchViews], private def evaluateInternal(event: ElasticSearchViewEvent): IO[ImportStatus] = { implicit val s: Subject = event.subject val cRev = event.rev - 1 + val project = projectMapper.map(event.project) event match { - case e: ElasticSearchViewCreated => views(event.uuid).flatMap(_.create(e.id, e.project, e.value)) - case e: ElasticSearchViewUpdated => views(event.uuid).flatMap(_.update(e.id, e.project, cRev, e.value)) - case e: ElasticSearchViewDeprecated => views(event.uuid).flatMap(_.deprecate(e.id, e.project, cRev)) - case e: ElasticSearchViewUndeprecated => views(event.uuid).flatMap(_.undeprecate(e.id, e.project, cRev)) + case e: ElasticSearchViewCreated => views(event.uuid).flatMap(_.create(e.id, project, e.value)) + case e: ElasticSearchViewUpdated => views(event.uuid).flatMap(_.update(e.id, project, cRev, e.value)) + case e: ElasticSearchViewDeprecated => views(event.uuid).flatMap(_.deprecate(e.id, project, cRev)) + case e: ElasticSearchViewUndeprecated => views(event.uuid).flatMap(_.undeprecate(e.id, project, cRev)) case _: ElasticSearchViewTagAdded => IO.unit // TODO: Check if this is correct } }.redeemWith( @@ -62,6 +66,7 @@ object ElasticSearchViewProcessor { def apply( fetchContext: FetchContext, rcr: ResolverContextResolution, + projectMapper: ProjectMapper, config: EventLogConfig, clock: EventClock, xas: Transactors @@ -90,7 +95,7 @@ object ElasticSearchViewProcessor { files.defaultSettings, clock )(jsonLdApi, UUIDF.fixed(uuid)) - } yield new ElasticSearchViewProcessor(views, clock) + } yield new ElasticSearchViewProcessor(views, projectMapper, clock) } } diff --git a/ship/src/test/resources/config/project-mapping.conf b/ship/src/test/resources/config/project-mapping.conf index bf33645629..3847c1e21f 100644 --- a/ship/src/test/resources/config/project-mapping.conf +++ b/ship/src/test/resources/config/project-mapping.conf @@ -1,5 +1,5 @@ ship { project-mapping = { - "private/mmb": "obp/reference" + "private/mmb": "obp/reference", } } \ No newline at end of file From 224a0b2d68cff8b5bae7e169bee486b241c97da4 Mon Sep 17 00:00:00 2001 From: Oliver <20188437+olivergrabinski@users.noreply.github.com> Date: Tue, 2 Apr 2024 15:43:36 +0200 Subject: [PATCH 3/7] scalafmtAll --- .../test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala index 4e61cdd909..08093c43d5 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala @@ -9,7 +9,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.resources.Resources import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors import ch.epfl.bluebrain.nexus.delta.sourcing.model.EntityType import ch.epfl.bluebrain.nexus.delta.sourcing.offset.Offset -import ch.epfl.bluebrain.nexus.delta.sourcing.postgres.Doobie.{PostgresPassword, PostgresUser, transactors} +import ch.epfl.bluebrain.nexus.delta.sourcing.postgres.Doobie.{transactors, PostgresPassword, PostgresUser} import ch.epfl.bluebrain.nexus.ship.ImportReport.Count import ch.epfl.bluebrain.nexus.ship.RunShipSuite.clearDB import ch.epfl.bluebrain.nexus.testkit.config.SystemPropertyOverride From ec25ede6f3654bddf59e091f416b39e1e2d10b3d Mon Sep 17 00:00:00 2001 From: Oliver <20188437+olivergrabinski@users.noreply.github.com> Date: Tue, 2 Apr 2024 15:58:43 +0200 Subject: [PATCH 4/7] Remove option usage --- .../scala/ch/epfl/bluebrain/nexus/ship/ProjectMapper.scala | 6 +++--- .../ch/epfl/bluebrain/nexus/ship/config/ShipConfig.scala | 2 +- .../epfl/bluebrain/nexus/ship/config/ShipConfigSuite.scala | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/ProjectMapper.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/ProjectMapper.scala index cfd5c52b3b..c3fc1f4629 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/ProjectMapper.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/ProjectMapper.scala @@ -9,11 +9,11 @@ trait ProjectMapper { object ProjectMapper { - def apply(projectMapping: Option[ProjectMapping]): ProjectMapper = + def apply(projectMapping: ProjectMapping): ProjectMapper = (project: ProjectRef) => projectMapping match { - case Some(mapping) => mapping.getOrElse(project, project) - case None => project + case m if m.isEmpty => project + case mapping => mapping.getOrElse(project, project) } } diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfig.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfig.scala index 7107a279ed..c643673914 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfig.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfig.scala @@ -19,7 +19,7 @@ final case class ShipConfig( database: DatabaseConfig, eventLog: EventLogConfig, organizations: OrganizationCreationConfig, - projectMapping: Option[ProjectMapping], + projectMapping: ProjectMapping = Map.empty, serviceAccount: ServiceAccountConfig ) diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigSuite.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigSuite.scala index 7454e8e390..97974894e8 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigSuite.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigSuite.scala @@ -28,7 +28,7 @@ class ShipConfigSuite extends NexusSuite { for { externalConfigPath <- loader.absolutePath("config/project-mapping.conf") mapping = ShipConfig.load(Some(Path(externalConfigPath))).map(_.projectMapping) - _ <- mapping.assertEquals(Some(expected)) + _ <- mapping.assertEquals(expected) } yield () } From ec56f7e876555e4ef48d14af9a8b92e81089224b Mon Sep 17 00:00:00 2001 From: Oliver <20188437+olivergrabinski@users.noreply.github.com> Date: Wed, 3 Apr 2024 09:50:52 +0200 Subject: [PATCH 5/7] Update docker-compose with optional way to run the ship --- tests/docker/config/external-ship.conf | 15 +++++++++++++ tests/docker/config/nginx.conf | 18 +++++++++++++++ tests/docker/docker-compose.yml | 31 ++++++++++++++++++++++++++ 3 files changed, 64 insertions(+) create mode 100644 tests/docker/config/external-ship.conf create mode 100644 tests/docker/config/nginx.conf diff --git a/tests/docker/config/external-ship.conf b/tests/docker/config/external-ship.conf new file mode 100644 index 0000000000..266d876e24 --- /dev/null +++ b/tests/docker/config/external-ship.conf @@ -0,0 +1,15 @@ +ship { + + database.access.host = "postgres" + +// organizations { +// values { +// example = "An example org" +// } +// } + +// project-mapping { +// "org/proj": "neworg/newproject" +// } + +} \ No newline at end of file diff --git a/tests/docker/config/nginx.conf b/tests/docker/config/nginx.conf new file mode 100644 index 0000000000..46ff6753f8 --- /dev/null +++ b/tests/docker/config/nginx.conf @@ -0,0 +1,18 @@ +server { + listen 80; + server_name localhost; + resolver 127.0.0.11; + + location / { + proxy_pass http://web:8000; + } + location /v1 { + proxy_pass http://delta:8080; + } + location /blazegraph { + proxy_pass http://blazegraph:9999; + } + location /elasticsearch/ { + proxy_pass http://elasticsearch:9200/; + } +} \ No newline at end of file diff --git a/tests/docker/docker-compose.yml b/tests/docker/docker-compose.yml index 0107119452..7588e72ea1 100644 --- a/tests/docker/docker-compose.yml +++ b/tests/docker/docker-compose.yml @@ -171,3 +171,34 @@ services: default: # Set the container IP address in the 10.0.2.0/24 subnet ipv4_address: 10.0.2.20 + +# fusion: +# depends_on: +# - delta +# image: bluebrain/nexus-web:1.9.9 +# environment: +# BASE_PATH: "/" +# HOST_NAME: "http://localhost" +# API_ENDPOINT: "http://localhost/v1" + +# router: +# depends_on: +# - web +# image: nginx:stable +# ports: +# - target: 80 +# published: 80 +# mode: host +# deploy: +# mode: global +# volumes: +# - ./config/nginx.conf:/etc/nginx/conf.d/default.conf + +# ship: +# depends_on: +# - postgres +# image: bluebrain/nexus-ship:latest +# command: run --file "/tmp/export.json" --config "/config/external-ship.conf" +# volumes: +# - /tmp/ship:/tmp # put the export file in /tmp/ship locally +# - ./config:/config \ No newline at end of file From e356d2d2c0c7ede8f1b9e05fffc229018a95e8eb Mon Sep 17 00:00:00 2001 From: Oliver <20188437+olivergrabinski@users.noreply.github.com> Date: Wed, 3 Apr 2024 10:14:46 +0200 Subject: [PATCH 6/7] Add test --- .../resources/config/project-mapping.conf | 1 + .../bluebrain/nexus/ship/RunShipSuite.scala | 22 +++++++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/ship/src/test/resources/config/project-mapping.conf b/ship/src/test/resources/config/project-mapping.conf index 3847c1e21f..df53e1f0f5 100644 --- a/ship/src/test/resources/config/project-mapping.conf +++ b/ship/src/test/resources/config/project-mapping.conf @@ -1,5 +1,6 @@ ship { project-mapping = { + "public/sscx": "obp/somato" "private/mmb": "obp/reference", } } \ No newline at end of file diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala index 08093c43d5..5e84b85639 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala @@ -11,7 +11,7 @@ import ch.epfl.bluebrain.nexus.delta.sourcing.model.EntityType import ch.epfl.bluebrain.nexus.delta.sourcing.offset.Offset import ch.epfl.bluebrain.nexus.delta.sourcing.postgres.Doobie.{transactors, PostgresPassword, PostgresUser} import ch.epfl.bluebrain.nexus.ship.ImportReport.Count -import ch.epfl.bluebrain.nexus.ship.RunShipSuite.clearDB +import ch.epfl.bluebrain.nexus.ship.RunShipSuite.{check, clearDB} import ch.epfl.bluebrain.nexus.testkit.config.SystemPropertyOverride import ch.epfl.bluebrain.nexus.testkit.mu.NexusSuite import ch.epfl.bluebrain.nexus.testkit.postgres.PostgresContainer @@ -61,6 +61,18 @@ class RunShipSuite extends NexusSuite with RunShipSuite.Fixture { } yield () } + test("Import with project mapping") { + for { + externalConfigPath <- loader.absolutePath("config/project-mapping.conf").map(x => Some(Path(x))) + importFileWithTwoProjects <- asPath("import/import.json") + _ <- new RunShip().run(importFileWithTwoProjects, externalConfigPath, Offset.start) + _ <- check(xas).map { projects => + assert(projects.size == 1) + assert(projects.contains(("obp", "somato"))) + } + } yield () + } + private def asPath(path: String): IO[Path] = { ClasspathResourceLoader().absolutePath(path).map(Path(_)) } @@ -77,6 +89,11 @@ object RunShipSuite { | DELETE FROM scoped_events; DELETE FROM scoped_states; |""".stripMargin.update.run.void.transact(xas.write) + def check(xas: Transactors) = + sql""" + | SELECT DISTINCT org, project FROM scoped_events; + """.stripMargin.query[(String, String)].to[List].transact(xas.read) + trait Fixture { self: CatsEffectSuite => private def initConfig(postgres: PostgresContainer) = @@ -84,7 +101,8 @@ object RunShipSuite { "ship.database.access.host" -> postgres.getHost, "ship.database.access.port" -> postgres.getMappedPort(5432).toString, "ship.database.tables-autocreate" -> "true", - "ship.organizations.values.public" -> "The public organization" + "ship.organizations.values.public" -> "The public organization", + "ship.organizations.values.obp" -> "The OBP organization" ) private val resource: Resource[IO, Transactors] = transactors( From cf3b0dda4fc3a98fa37f526c513c18a815c66fee Mon Sep 17 00:00:00 2001 From: Oliver <20188437+olivergrabinski@users.noreply.github.com> Date: Wed, 3 Apr 2024 10:26:55 +0200 Subject: [PATCH 7/7] Add test --- ship/src/test/resources/config/project-mapping-sscx.conf | 5 +++++ ship/src/test/resources/config/project-mapping.conf | 3 +-- .../scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 ship/src/test/resources/config/project-mapping-sscx.conf diff --git a/ship/src/test/resources/config/project-mapping-sscx.conf b/ship/src/test/resources/config/project-mapping-sscx.conf new file mode 100644 index 0000000000..ebab053fdc --- /dev/null +++ b/ship/src/test/resources/config/project-mapping-sscx.conf @@ -0,0 +1,5 @@ +ship { + project-mapping = { + "public/sscx": "obp/somato" + } +} \ No newline at end of file diff --git a/ship/src/test/resources/config/project-mapping.conf b/ship/src/test/resources/config/project-mapping.conf index df53e1f0f5..bf33645629 100644 --- a/ship/src/test/resources/config/project-mapping.conf +++ b/ship/src/test/resources/config/project-mapping.conf @@ -1,6 +1,5 @@ ship { project-mapping = { - "public/sscx": "obp/somato" - "private/mmb": "obp/reference", + "private/mmb": "obp/reference" } } \ No newline at end of file diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala index 5e84b85639..117241e35c 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala @@ -61,9 +61,9 @@ class RunShipSuite extends NexusSuite with RunShipSuite.Fixture { } yield () } - test("Import with project mapping") { + test("Import and map public/sscx to obp/somato") { for { - externalConfigPath <- loader.absolutePath("config/project-mapping.conf").map(x => Some(Path(x))) + externalConfigPath <- loader.absolutePath("config/project-mapping-sscx.conf").map(x => Some(Path(x))) importFileWithTwoProjects <- asPath("import/import.json") _ <- new RunShip().run(importFileWithTwoProjects, externalConfigPath, Offset.start) _ <- check(xas).map { projects =>