From bf6fe3ee034c8444b4d2d3d617bfedd06c313984 Mon Sep 17 00:00:00 2001 From: Oliver <20188437+olivergrabinski@users.noreply.github.com> Date: Mon, 15 Apr 2024 14:22:10 +0200 Subject: [PATCH 1/8] ES/BG view init --- build.sbt | 1 + ship/src/main/resources/ship-default.conf | 17 +++ .../epfl/bluebrain/nexus/ship/RunShip.scala | 2 +- .../nexus/ship/config/ShipConfig.scala | 1 + .../nexus/ship/config/ViewDefaults.scala | 16 +++ .../ship/projects/ProjectProcessor.scala | 34 +++-- .../ship/views/BlazegraphViewProcessor.scala | 38 ++--- .../ship/views/CompositeViewProcessor.scala | 53 +++---- .../views/ElasticSearchViewProcessor.scala | 49 +++---- .../nexus/ship/views/ViewWiring.scala | 133 ++++++++++++++++++ ship/src/test/resources/config/views.conf | 5 + .../bluebrain/nexus/ship/RunShipSuite.scala | 2 +- .../nexus/ship/config/ShipConfigSuite.scala | 17 +++ 13 files changed, 276 insertions(+), 92 deletions(-) create mode 100644 ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ViewDefaults.scala create mode 100644 ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ViewWiring.scala create mode 100644 ship/src/test/resources/config/views.conf diff --git a/build.sbt b/build.sbt index c62c0df1e5..b052802188 100755 --- a/build.sbt +++ b/build.sbt @@ -751,6 +751,7 @@ lazy val ship = project compositeViewsPlugin % "compile->compile", elasticsearchPlugin % "compile->compile", storagePlugin % "compile->compile;test->test", + searchPlugin % "compile->compile", tests % "test->compile;test->test" ) .settings( diff --git a/ship/src/main/resources/ship-default.conf b/ship/src/main/resources/ship-default.conf index 5946c51340..a9a850aa89 100644 --- a/ship/src/main/resources/ship-default.conf +++ b/ship/src/main/resources/ship-default.conf @@ -54,6 +54,23 @@ ship { } } + view-defaults { + elasticsearch { + name = "Default Elasticsearch view" + description = "An Elasticsearch view of all resources in the project." + } + + blazegraph { + name = "Default Sparql view" + description = "A Sparql view of all resources in the project." + } + + search { + name = "Default global search view" + description = "An Elasticsearch view of configured resources for the global search." + } + } + # Service account configuration for internal operations service-account { subject: "delta" diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala index 5c3e29454e..6c81b3ed74 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala @@ -62,7 +62,7 @@ trait RunShip { rcr = ContextWiring.resolverContextResolution(fetchResource, fetchContext, remoteContextResolution, eventLogConfig, eventClock, xas) schemaImports = SchemaWiring.schemaImports(fetchResource, fetchSchema, fetchContext, eventLogConfig, eventClock, xas) // Processors - projectProcessor <- ProjectProcessor(fetchActiveOrg, projectMapper, eventLogConfig, eventClock, xas)(baseUri) + projectProcessor <- ProjectProcessor(fetchActiveOrg, fetchContext, rcr, projectMapper, config, eventClock, xas)(baseUri, jsonLdApi) resolverProcessor = ResolverProcessor(fetchContext, projectMapper, eventLogConfig, eventClock, xas) schemaProcessor = SchemaProcessor(schemaLog, fetchContext, schemaImports, rcr, projectMapper, eventClock) resourceProcessor = ResourceProcessor(resourceLog, rcr, projectMapper, fetchContext, eventClock) diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfig.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfig.scala index 5ce2a14eee..eee6113a39 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfig.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfig.scala @@ -25,6 +25,7 @@ final case class ShipConfig( eventLog: EventLogConfig, organizations: OrganizationCreationConfig, projectMapping: ProjectMapping = Map.empty, + viewDefaults: ViewDefaults, serviceAccount: ServiceAccountConfig ) diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ViewDefaults.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ViewDefaults.scala new file mode 100644 index 0000000000..c2f7b4ae45 --- /dev/null +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ViewDefaults.scala @@ -0,0 +1,16 @@ +package ch.epfl.bluebrain.nexus.ship.config + +import ch.epfl.bluebrain.nexus.delta.sdk.Defaults +import pureconfig.ConfigReader +import pureconfig.generic.semiauto.deriveReader + +case class ViewDefaults( + elasticsearch: Defaults, + blazegraph: Defaults, + search: Defaults +) + +object ViewDefaults { + implicit val viewDefaultsConfigReader: ConfigReader[ViewDefaults] = + deriveReader[ViewDefaults] +} diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala index 4a6f5ce81a..b716b4b418 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala @@ -2,27 +2,32 @@ package ch.epfl.bluebrain.nexus.ship.projects import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.Logger +import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi import ch.epfl.bluebrain.nexus.delta.sdk.ScopeInitializer import ch.epfl.bluebrain.nexus.delta.sdk.model.BaseUri import ch.epfl.bluebrain.nexus.delta.sdk.organizations.FetchActiveOrganization import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ProjectEvent._ import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ProjectRejection.NotFound import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.{ApiMappings, ProjectEvent, ProjectFields, ProjectRejection} -import ch.epfl.bluebrain.nexus.delta.sdk.projects.{Projects, ProjectsImpl, ValidateProjectDeletion} +import ch.epfl.bluebrain.nexus.delta.sdk.projects.{FetchContext, Projects, ProjectsImpl, ValidateProjectDeletion} +import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors -import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject import ch.epfl.bluebrain.nexus.delta.sourcing.model.{EntityType, ProjectRef} +import ch.epfl.bluebrain.nexus.ship.config.ShipConfig import ch.epfl.bluebrain.nexus.ship.error.ShipError.ProjectDeletionIsNotAllowed import ch.epfl.bluebrain.nexus.ship.projects.ProjectProcessor.logger -import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, EventUUIDF, ImportStatus, ProjectMapper} +import ch.epfl.bluebrain.nexus.ship.views.ViewWiring +import ch.epfl.bluebrain.nexus.ship._ import io.circe.Decoder final class ProjectProcessor private ( projects: Projects, projectMapper: ProjectMapper, clock: EventClock, - uuidF: EventUUIDF + uuidF: EventUUIDF, + scopeInitializer: ScopeInitializer ) extends EventProcessor[ProjectEvent] { override def resourceType: EntityType = Projects.entityType @@ -44,7 +49,8 @@ final class ProjectProcessor private ( event match { case ProjectCreated(_, _, _, _, _, description, apiMappings, base, vocab, enforceSchema, _, _) => val fields = ProjectFields(description, apiMappings, Some(base), Some(vocab), enforceSchema) - projects.create(projectRef, fields) + projects.create(projectRef, fields) >> + scopeInitializer.initializeProject(projectRef) case ProjectUpdated(_, _, _, _, _, description, apiMappings, base, vocab, enforceSchema, _, _) => val fields = ProjectFields(description, apiMappings, Some(base), Some(vocab), enforceSchema) projects.update(projectRef, cRev, fields) @@ -70,15 +76,23 @@ object ProjectProcessor { private val logger = Logger[ProjectProcessor] def apply( fetchActiveOrg: FetchActiveOrganization, + fetchContext: FetchContext, + rcr: ResolverContextResolution, projectMapper: ProjectMapper, - config: EventLogConfig, + config: ShipConfig, clock: EventClock, xas: Transactors )(implicit - base: BaseUri + base: BaseUri, + jsonLdApi: JsonLdApi ): IO[ProjectProcessor] = for { - uuidF <- EventUUIDF.init() + uuidF <- EventUUIDF.init() + esViewUuid <- UUIDF.random() + bgViewUuid <- UUIDF.random() + esViews <- ViewWiring.esViews(fetchContext, rcr, config.eventLog, clock, xas).flatMap(_(esViewUuid)) + bgViews <- ViewWiring.bgViews(fetchContext, rcr, config.eventLog, clock, xas)(jsonLdApi)(bgViewUuid) + initializer = ViewWiring.viewInitializer(esViews, bgViews, config) } yield { val disableDeletion: ValidateProjectDeletion = (p: ProjectRef) => IO.raiseError(ProjectDeletionIsNotAllowed(p)) val projects = ProjectsImpl( @@ -86,10 +100,10 @@ object ProjectProcessor { disableDeletion, ScopeInitializer.noop, ApiMappings.empty, - config, + config.eventLog, xas, clock )(base, uuidF) - new ProjectProcessor(projects, projectMapper, clock, uuidF) + new ProjectProcessor(projects, projectMapper, clock, uuidF, initializer) } } diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/BlazegraphViewProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/BlazegraphViewProcessor.scala index dddbe98488..d814fa4268 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/BlazegraphViewProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/BlazegraphViewProcessor.scala @@ -2,11 +2,10 @@ package ch.epfl.bluebrain.nexus.ship.views import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.Logger -import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF +import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.BlazegraphViews import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.BlazegraphViewEvent._ import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.BlazegraphViewRejection.{IncorrectRev, ResourceAlreadyExists} -import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.{defaultViewId, BlazegraphViewEvent, BlazegraphViewValue, ViewResource} -import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.{BlazegraphViews, ValidateBlazegraphView} +import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.{defaultViewId, BlazegraphViewEvent} import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext @@ -45,16 +44,24 @@ class BlazegraphViewProcessor private ( event match { case e: BlazegraphViewCreated => e.id match { - case id if id == defaultViewId => views(event.uuid).flatMap(_.create(e.id, project, e.value)) + case id if id == defaultViewId => IO.unit // the default view is created on project creation case _ => views(event.uuid).flatMap(_.create(e.id, project, e.source)) } case e: BlazegraphViewUpdated => e.id match { - case id if id == defaultViewId => views(event.uuid).flatMap(_.update(e.id, project, cRev, e.value)) + case id if id == defaultViewId => IO.unit case _ => views(event.uuid).flatMap(_.update(e.id, project, cRev, e.source)) } - case e: BlazegraphViewDeprecated => views(event.uuid).flatMap(_.deprecate(e.id, project, cRev)) - case e: BlazegraphViewUndeprecated => views(event.uuid).flatMap(_.undeprecate(e.id, project, cRev)) + case e: BlazegraphViewDeprecated => + e.id match { + case id if id == defaultViewId => IO.unit + case _ => views(event.uuid).flatMap(_.deprecate(e.id, project, cRev)) + } + case e: BlazegraphViewUndeprecated => + e.id match { + case id if id == defaultViewId => IO.unit + case _ => views(event.uuid).flatMap(_.undeprecate(e.id, project, cRev)) + } case _: BlazegraphViewTagAdded => IO.unit // TODO: Can we tag? } }.redeemWith( @@ -81,22 +88,7 @@ object BlazegraphViewProcessor { )(implicit jsonLdApi: JsonLdApi ): BlazegraphViewProcessor = { - val noValidation = new ValidateBlazegraphView { - override def apply(value: BlazegraphViewValue): IO[Unit] = IO.unit - } - val prefix = "nexus" // TODO: use the config? - - val views = (uuid: UUID) => - BlazegraphViews( - fetchContext, - rcr, - noValidation, - (_: ViewResource) => IO.unit, - config, - prefix, - xas, - clock - )(jsonLdApi, UUIDF.fixed(uuid)) + val views = ViewWiring.bgViews(fetchContext, rcr, config, clock, xas) new BlazegraphViewProcessor(views, projectMapper, clock) } diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/CompositeViewProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/CompositeViewProcessor.scala index 6d0714a09c..90c6c5b49f 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/CompositeViewProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/CompositeViewProcessor.scala @@ -2,11 +2,11 @@ package ch.epfl.bluebrain.nexus.ship.views import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.Logger -import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF +import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.CompositeViews +import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewEvent import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewEvent._ import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewRejection.{IncorrectRev, ResourceAlreadyExists} -import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.{CompositeViewEvent, CompositeViewValue} -import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.{CompositeViews, ValidateCompositeView} +import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext @@ -20,7 +20,6 @@ import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, ImportStatus, P import io.circe.Decoder import java.util.UUID -import scala.concurrent.duration.DurationInt class CompositeViewProcessor(views: UUID => IO[CompositeViews], projectMapper: ProjectMapper, clock: EventClock) extends EventProcessor[CompositeViewEvent] { @@ -39,12 +38,31 @@ class CompositeViewProcessor(views: UUID => IO[CompositeViews], projectMapper: P implicit val c: Caller = Caller(s, Set.empty) val cRev = event.rev - 1 val project = projectMapper.map(event.project) + val searchViewId = nxv + "searchView" + event match { - case e: CompositeViewCreated => views(event.uuid).flatMap(_.create(project, e.source)) - case e: CompositeViewUpdated => views(event.uuid).flatMap(_.update(e.id, project, cRev, e.source)) - case e: CompositeViewDeprecated => views(event.uuid).flatMap(_.deprecate(e.id, project, cRev)) - case e: CompositeViewUndeprecated => views(event.uuid).flatMap(_.undeprecate(e.id, project, cRev)) - case _: CompositeViewTagAdded => IO.unit // TODO: Can/should we tag? + case e: CompositeViewCreated => + e.id match { + case id if id == searchViewId => IO.unit // The search view is created upon project creation + case _ => views(event.uuid).flatMap(_.create(project, e.source)) + } + case e: CompositeViewUpdated => + e.id match { + case id if id == searchViewId => IO.unit + case _ => views(event.uuid).flatMap(_.update(e.id, project, cRev, e.source)) + } + case e: CompositeViewDeprecated => + e.id match { + case id if id == searchViewId => IO.unit + case _ => views(event.uuid).flatMap(_.deprecate(e.id, project, cRev)) + } + case e: CompositeViewUndeprecated => + e.id match { + case id if id == searchViewId => IO.unit + case _ => views(event.uuid).flatMap(_.undeprecate(e.id, project, cRev)) + } + case _: CompositeViewTagAdded => + IO.unit // TODO: Can/should we tag? } }.redeemWith( { @@ -70,23 +88,8 @@ object CompositeViewProcessor { )(implicit jsonLdApi: JsonLdApi ): CompositeViewProcessor = { - val noValidation = new ValidateCompositeView { - override def apply(uuid: UUID, value: CompositeViewValue): IO[Unit] = IO.unit - } - - val views = (uuid: UUID) => - CompositeViews( - fetchContext, - rcr, - noValidation, - 3.seconds, - config, - xas, - clock - )(jsonLdApi, UUIDF.fixed(uuid)) - + val views = ViewWiring.cvViews(fetchContext, rcr, config, clock, xas) new CompositeViewProcessor(views, projectMapper, clock) - } } diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ElasticSearchViewProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ElasticSearchViewProcessor.scala index 43b017ab35..fcac6098c1 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ElasticSearchViewProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ElasticSearchViewProcessor.scala @@ -2,16 +2,14 @@ package ch.epfl.bluebrain.nexus.ship.views import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.Logger -import ch.epfl.bluebrain.nexus.delta.kernel.utils.{ClasspathResourceLoader, UUIDF} +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.ElasticSearchViews import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.ElasticSearchViewEvent._ import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.ElasticSearchViewRejection.{IncorrectRev, ResourceAlreadyExists} -import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.{defaultViewId, ElasticSearchFiles, ElasticSearchViewEvent, ElasticSearchViewValue} -import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.{ElasticSearchViews, ValidateElasticSearchView} +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.{defaultViewId, ElasticSearchViewEvent} import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution -import ch.epfl.bluebrain.nexus.delta.sdk.views.IndexingRev import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig import ch.epfl.bluebrain.nexus.delta.sourcing.model.EntityType @@ -46,16 +44,24 @@ class ElasticSearchViewProcessor private ( event match { case e: ElasticSearchViewCreated => e.id match { - case id if id == defaultViewId => views(event.uuid).flatMap(_.create(e.id, project, e.value)) + case id if id == defaultViewId => IO.unit // the default view is created on project creation case _ => views(event.uuid).flatMap(_.create(e.id, project, e.source)) } case e: ElasticSearchViewUpdated => e.id match { - case id if id == defaultViewId => views(event.uuid).flatMap(_.update(e.id, project, cRev, e.value)) + case id if id == defaultViewId => IO.unit case _ => views(event.uuid).flatMap(_.update(e.id, project, cRev, e.source)) } - case e: ElasticSearchViewDeprecated => views(event.uuid).flatMap(_.deprecate(e.id, project, cRev)) - case e: ElasticSearchViewUndeprecated => views(event.uuid).flatMap(_.undeprecate(e.id, project, cRev)) + case e: ElasticSearchViewDeprecated => + e.id match { + case id if id == defaultViewId => IO.unit + case _ => views(event.uuid).flatMap(_.deprecate(e.id, project, cRev)) + } + case e: ElasticSearchViewUndeprecated => + e.id match { + case id if id == defaultViewId => IO.unit + case _ => views(event.uuid).flatMap(_.undeprecate(e.id, project, cRev)) + } case _: ElasticSearchViewTagAdded => IO.unit // TODO: Check if this is correct } }.redeemWith( @@ -82,30 +88,9 @@ object ElasticSearchViewProcessor { xas: Transactors )(implicit jsonLdApi: JsonLdApi - ): IO[ElasticSearchViewProcessor] = { - implicit val loader: ClasspathResourceLoader = ClasspathResourceLoader.withContext(getClass) - - val noValidation = new ValidateElasticSearchView { - override def apply(uuid: UUID, indexingRev: IndexingRev, v: ElasticSearchViewValue): IO[Unit] = IO.unit + ): IO[ElasticSearchViewProcessor] = + ViewWiring.esViews(fetchContext, rcr, config, clock, xas).map { views => + new ElasticSearchViewProcessor(views, projectMapper, clock) } - val prefix = "nexus" // TODO: use the config? - val esFiles = ElasticSearchFiles.mk(loader) - - for { - files <- esFiles - views = (uuid: UUID) => - ElasticSearchViews( - fetchContext, - rcr, - noValidation, - config, - prefix, - xas, - files.defaultMapping, - files.defaultSettings, - clock - )(jsonLdApi, UUIDF.fixed(uuid)) - } yield new ElasticSearchViewProcessor(views, projectMapper, clock) - } } diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ViewWiring.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ViewWiring.scala new file mode 100644 index 0000000000..d87474e6c4 --- /dev/null +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ViewWiring.scala @@ -0,0 +1,133 @@ +package ch.epfl.bluebrain.nexus.ship.views + +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.kernel.utils.{ClasspathResourceLoader, UUIDF} +import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.{BlazegraphViewValue, ViewResource} +import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.{BlazegraphScopeInitialization, BlazegraphViews, ValidateBlazegraphView} +import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewValue +import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.{CompositeViews, ValidateCompositeView} +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.{ElasticSearchFiles, ElasticSearchViewValue} +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.{ElasticSearchScopeInitialization, ElasticSearchViews, ValidateElasticSearchView} +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi +import ch.epfl.bluebrain.nexus.delta.sdk.error.ServiceError +import ch.epfl.bluebrain.nexus.delta.sdk.projects.{FetchContext, ScopeInitializationErrorStore} +import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution +import ch.epfl.bluebrain.nexus.delta.sdk.views.IndexingRev +import ch.epfl.bluebrain.nexus.delta.sdk.{ScopeInitialization, ScopeInitializer} +import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors +import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{EntityType, ProjectRef} +import ch.epfl.bluebrain.nexus.ship.EventClock +import ch.epfl.bluebrain.nexus.ship.config.ShipConfig + +import java.util.UUID +import scala.concurrent.duration.DurationInt + +object ViewWiring { + + def esViews( + fetchContext: FetchContext, + rcr: ResolverContextResolution, + config: EventLogConfig, + clock: EventClock, + xas: Transactors + )(implicit jsonLdApi: JsonLdApi) = { + implicit val loader: ClasspathResourceLoader = ClasspathResourceLoader.withContext(getClass) + val prefix = "nexus" // TODO: use the config? + + val noValidation = new ValidateElasticSearchView { + override def apply(uuid: UUID, indexingRev: IndexingRev, v: ElasticSearchViewValue): IO[Unit] = IO.unit + } + + ElasticSearchFiles.mk(loader).map { files => (uuid: UUID) => + ElasticSearchViews( + fetchContext, + rcr, + noValidation, + config, + prefix, + xas, + files.defaultMapping, + files.defaultSettings, + clock + )(jsonLdApi, UUIDF.fixed(uuid)) + } + } + + def bgViews( + fetchContext: FetchContext, + rcr: ResolverContextResolution, + config: EventLogConfig, + clock: EventClock, + xas: Transactors + )(implicit jsonLdApi: JsonLdApi) = { + val noValidation = new ValidateBlazegraphView { + override def apply(value: BlazegraphViewValue): IO[Unit] = IO.unit + } + val prefix = "nexus" // TODO: use the config? + (uuid: UUID) => + BlazegraphViews( + fetchContext, + rcr, + noValidation, + (_: ViewResource) => IO.unit, + config, + prefix, + xas, + clock + )(jsonLdApi, UUIDF.fixed(uuid)) + } + + def cvViews( + fetchContext: FetchContext, + rcr: ResolverContextResolution, + config: EventLogConfig, + clock: EventClock, + xas: Transactors + )(implicit jsonLdApi: JsonLdApi) = { + val noValidation = new ValidateCompositeView { + override def apply(uuid: UUID, value: CompositeViewValue): IO[Unit] = IO.unit + } + (uuid: UUID) => + CompositeViews( + fetchContext, + rcr, + noValidation, + 3.seconds, // TODO: use the config? + config, + xas, + clock + )(jsonLdApi, UUIDF.fixed(uuid)) + } + + def viewInitializer( + esViews: ElasticSearchViews, + bgViews: BlazegraphViews, + config: ShipConfig + ) = { + val viewInits = Set.empty[ScopeInitialization] + + new ElasticSearchScopeInitialization( + esViews, + config.serviceAccount.value, + config.viewDefaults.elasticsearch + ) + new BlazegraphScopeInitialization( + bgViews, + config.serviceAccount.value, + config.viewDefaults.blazegraph + ) + ScopeInitializer(viewInits, noopErrorStore) + } + + def noopErrorStore: ScopeInitializationErrorStore = new ScopeInitializationErrorStore { + override def save( + entityType: EntityType, + project: ProjectRef, + e: ServiceError.ScopeInitializationFailed + ): IO[Unit] = IO.unit + + override def fetch: IO[List[ScopeInitializationErrorStore.ScopeInitErrorRow]] = IO.pure(List.empty) + + override def delete(project: ProjectRef): IO[Unit] = IO.unit + } + +} diff --git a/ship/src/test/resources/config/views.conf b/ship/src/test/resources/config/views.conf new file mode 100644 index 0000000000..2a98111bb3 --- /dev/null +++ b/ship/src/test/resources/config/views.conf @@ -0,0 +1,5 @@ +ship { + + + +} \ No newline at end of file diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala index 6fe9cf147d..e9f39c34ad 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala @@ -9,7 +9,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.resources.Resources import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors import ch.epfl.bluebrain.nexus.delta.sourcing.model.EntityType import ch.epfl.bluebrain.nexus.delta.sourcing.offset.Offset -import ch.epfl.bluebrain.nexus.delta.sourcing.postgres.Doobie.{PostgresPassword, PostgresUser, transactors} +import ch.epfl.bluebrain.nexus.delta.sourcing.postgres.Doobie.{transactors, PostgresPassword, PostgresUser} import ch.epfl.bluebrain.nexus.ship.ImportReport.Count import ch.epfl.bluebrain.nexus.ship.RunShipSuite.{clearDB, expectedImportReport, getDistinctOrgProjects} import ch.epfl.bluebrain.nexus.testkit.config.SystemPropertyOverride diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigSuite.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigSuite.scala index c663dc240e..a77b1161e2 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigSuite.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigSuite.scala @@ -1,7 +1,9 @@ package ch.epfl.bluebrain.nexus.ship.config +import ch.epfl.bluebrain.nexus.delta.sdk.Defaults import ch.epfl.bluebrain.nexus.delta.sdk.model.BaseUri import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Label, ProjectRef} +import ch.epfl.bluebrain.nexus.ship.config.ShipConfigSuite.{defaultBgValues, defaultEsValues, defaultSearchValues} import ch.epfl.bluebrain.nexus.testkit.mu.NexusSuite import eu.timepit.refined.types.string.NonEmptyString import fs2.aws.s3.models.Models.BucketName @@ -16,6 +18,13 @@ class ShipConfigSuite extends NexusSuite { ShipConfig.load(None).map(_.baseUri).assertEquals(expectedBaseUri) } + test("The defaults (name/description) for views should be correct") { + val config = ShipConfig.load(None) + config.map(_.viewDefaults.elasticsearch).assertEquals(defaultEsValues) >> + config.map(_.viewDefaults.blazegraph).assertEquals(defaultBgValues) >> + config.map(_.viewDefaults.search).assertEquals(defaultSearchValues) + } + test("Default configuration should be overloaded by the external config") { val expectedBaseUri = BaseUri("https://bbp.epfl.ch", Label.unsafe("v1")) for { @@ -47,3 +56,11 @@ class ShipConfigSuite extends NexusSuite { } } + +object ShipConfigSuite { + private val defaultEsValues = + Defaults("Default Elasticsearch view", "An Elasticsearch view of all resources in the project.") + private val defaultBgValues = Defaults("Default Sparql view", "A Sparql view of all resources in the project.") + private val defaultSearchValues = + Defaults("Default global search view", "An Elasticsearch view of configured resources for the global search.") +} From fa9c24dd878c0abfea97f5ba952b95262e578f9c Mon Sep 17 00:00:00 2001 From: Oliver <20188437+olivergrabinski@users.noreply.github.com> Date: Mon, 15 Apr 2024 14:38:36 +0200 Subject: [PATCH 2/8] Add test --- .../bluebrain/nexus/ship/RunShipSuite.scala | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala index e9f39c34ad..c38a82359e 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala @@ -3,15 +3,17 @@ package ch.epfl.bluebrain.nexus.ship import cats.effect.{IO, Resource} import cats.syntax.all._ import ch.epfl.bluebrain.nexus.delta.kernel.utils.ClasspathResourceLoader +import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri +import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv import ch.epfl.bluebrain.nexus.delta.sdk.projects.Projects import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.Resolvers import ch.epfl.bluebrain.nexus.delta.sdk.resources.Resources import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors import ch.epfl.bluebrain.nexus.delta.sourcing.model.EntityType import ch.epfl.bluebrain.nexus.delta.sourcing.offset.Offset -import ch.epfl.bluebrain.nexus.delta.sourcing.postgres.Doobie.{transactors, PostgresPassword, PostgresUser} +import ch.epfl.bluebrain.nexus.delta.sourcing.postgres.Doobie.{PostgresPassword, PostgresUser, transactors} import ch.epfl.bluebrain.nexus.ship.ImportReport.Count -import ch.epfl.bluebrain.nexus.ship.RunShipSuite.{clearDB, expectedImportReport, getDistinctOrgProjects} +import ch.epfl.bluebrain.nexus.ship.RunShipSuite.{checkFor, clearDB, expectedImportReport, getDistinctOrgProjects} import ch.epfl.bluebrain.nexus.testkit.config.SystemPropertyOverride import ch.epfl.bluebrain.nexus.testkit.mu.NexusSuite import ch.epfl.bluebrain.nexus.testkit.postgres.PostgresContainer @@ -44,6 +46,15 @@ class RunShipSuite extends NexusSuite with RunShipSuite.Fixture { } yield () } + test("Run import and check for views") { + for { + importFile <- asPath("import/import.json") + _ <- RunShip.localShip.run(importFile, None).assertEquals(expectedImportReport) + _ <- checkFor("elasticsearch", nxv + "defaultElasticSearchIndex", xas).assertEquals(1) + _ <- checkFor("blazegraph", nxv + "defaultSparqlIndex", xas).assertEquals(1) + } yield () + } + test("Run import by providing the path to a directory") { for { importDirectory <- asPath("import/multi-part-import") @@ -95,6 +106,13 @@ object RunShipSuite { | SELECT DISTINCT org, project FROM scoped_events; """.stripMargin.query[(String, String)].to[List].transact(xas.read) + def checkFor(entityType: String, id: Iri, xas: Transactors): IO[Int] = + sql""" + | SELECT COUNT(*) FROM scoped_events + | WHERE type = $entityType + | AND id = ${id.toString} + """.stripMargin.query[Int].unique.transact(xas.read) + // The expected import report for the import.json file, as well as for the /import/multi-part-import directory val expectedImportReport: ImportReport = ImportReport( Offset.at(9999999L), From 5763c1b4f1ec0bca22ada68354b2b9189d464d2e Mon Sep 17 00:00:00 2001 From: Oliver <20188437+olivergrabinski@users.noreply.github.com> Date: Mon, 15 Apr 2024 14:38:41 +0200 Subject: [PATCH 3/8] Remove test --- .../nexus/ship/ShipIntegrationSpec.scala | 45 ------------------- 1 file changed, 45 deletions(-) diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/ShipIntegrationSpec.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/ShipIntegrationSpec.scala index 07fecd0164..a47f9c330f 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/ShipIntegrationSpec.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/ShipIntegrationSpec.scala @@ -139,29 +139,6 @@ class ShipIntegrationSpec extends BaseIntegrationSpec { thereShouldBeAView(project, bgView, bgViewJson) } - "transfer a search view" in { - val (project, _, _) = thereIsAProject() - val (searchView, searchViewJson) = thereIsASearchView(project) - - whenTheExportIsRunOnProject(project) - theOldProjectIsDeleted(project) - - weRunTheImporter(project) - weFixThePermissions(project) - - thereShouldBeAViewIgnoringUUID(project, searchView, searchViewJson) - } - - def thereIsASearchView(project: ProjectRef): (Iri, Json) = { - val searchView = nxv + "searchView" - val encodedView = UrlUtils.encode(searchView.toString) - val (viewJson, status) = deltaClient - .getJsonAndStatus(s"/views/${project.organization}/${project.project}/$encodedView", writer) - .accepted - status shouldEqual StatusCodes.OK - searchView -> viewJson - } - def thereShouldBeAView(project: ProjectRef, view: Iri, originalJson: Json): Assertion = { val encodedIri = UrlUtils.encode(view.toString) deltaClient @@ -174,28 +151,6 @@ class ShipIntegrationSpec extends BaseIntegrationSpec { .accepted } - def thereShouldBeAViewIgnoringUUID(project: ProjectRef, view: Iri, originalJson: Json): Assertion = { - val encodedIri = UrlUtils.encode(view.toString) - - import io.circe.optics.JsonPath.root - val ignoreSourceUUID = root.sources.each.at("_uuid").replace(None) - val ignoreProjectionUUID = root.projections.each.at("_uuid").replace(None) - val ignoreUUID = root.at("_uuid").replace(None) - - val filter = ignoreUUID andThen ignoreSourceUUID andThen ignoreProjectionUUID - - root.sources.`null` - - deltaClient - .get[Json](s"/views/${project.organization}/${project.project}/$encodedIri", writer) { (json, response) => - { - response.status shouldEqual StatusCodes.OK - filter(json) shouldEqual filter(originalJson) - } - } - .accepted - } - def thereIsABlazegraphView(project: ProjectRef): (Iri, Json) = { val simpleBgView = json"""{ "@type": "SparqlView", From 2f677a8e9c5e1e9eceff4f54c888e440b629c2da Mon Sep 17 00:00:00 2001 From: Oliver <20188437+olivergrabinski@users.noreply.github.com> Date: Mon, 15 Apr 2024 14:48:56 +0200 Subject: [PATCH 4/8] Revert "Remove test" This reverts commit 5763c1b4f1ec0bca22ada68354b2b9189d464d2e. --- .../nexus/ship/ShipIntegrationSpec.scala | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/ShipIntegrationSpec.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/ShipIntegrationSpec.scala index a47f9c330f..07fecd0164 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/ShipIntegrationSpec.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/ShipIntegrationSpec.scala @@ -139,6 +139,29 @@ class ShipIntegrationSpec extends BaseIntegrationSpec { thereShouldBeAView(project, bgView, bgViewJson) } + "transfer a search view" in { + val (project, _, _) = thereIsAProject() + val (searchView, searchViewJson) = thereIsASearchView(project) + + whenTheExportIsRunOnProject(project) + theOldProjectIsDeleted(project) + + weRunTheImporter(project) + weFixThePermissions(project) + + thereShouldBeAViewIgnoringUUID(project, searchView, searchViewJson) + } + + def thereIsASearchView(project: ProjectRef): (Iri, Json) = { + val searchView = nxv + "searchView" + val encodedView = UrlUtils.encode(searchView.toString) + val (viewJson, status) = deltaClient + .getJsonAndStatus(s"/views/${project.organization}/${project.project}/$encodedView", writer) + .accepted + status shouldEqual StatusCodes.OK + searchView -> viewJson + } + def thereShouldBeAView(project: ProjectRef, view: Iri, originalJson: Json): Assertion = { val encodedIri = UrlUtils.encode(view.toString) deltaClient @@ -151,6 +174,28 @@ class ShipIntegrationSpec extends BaseIntegrationSpec { .accepted } + def thereShouldBeAViewIgnoringUUID(project: ProjectRef, view: Iri, originalJson: Json): Assertion = { + val encodedIri = UrlUtils.encode(view.toString) + + import io.circe.optics.JsonPath.root + val ignoreSourceUUID = root.sources.each.at("_uuid").replace(None) + val ignoreProjectionUUID = root.projections.each.at("_uuid").replace(None) + val ignoreUUID = root.at("_uuid").replace(None) + + val filter = ignoreUUID andThen ignoreSourceUUID andThen ignoreProjectionUUID + + root.sources.`null` + + deltaClient + .get[Json](s"/views/${project.organization}/${project.project}/$encodedIri", writer) { (json, response) => + { + response.status shouldEqual StatusCodes.OK + filter(json) shouldEqual filter(originalJson) + } + } + .accepted + } + def thereIsABlazegraphView(project: ProjectRef): (Iri, Json) = { val simpleBgView = json"""{ "@type": "SparqlView", From ce572f4d9f76fdd8366125bd3f1d68e51f1c5234 Mon Sep 17 00:00:00 2001 From: Oliver <20188437+olivergrabinski@users.noreply.github.com> Date: Mon, 15 Apr 2024 14:51:06 +0200 Subject: [PATCH 5/8] Don't handle the search views right now --- build.sbt | 1 - ship/src/main/resources/ship-default.conf | 5 ---- .../nexus/ship/config/ViewDefaults.scala | 3 +- .../ship/views/CompositeViewProcessor.scala | 29 ++++--------------- .../bluebrain/nexus/ship/RunShipSuite.scala | 2 +- 5 files changed, 7 insertions(+), 33 deletions(-) diff --git a/build.sbt b/build.sbt index b052802188..c62c0df1e5 100755 --- a/build.sbt +++ b/build.sbt @@ -751,7 +751,6 @@ lazy val ship = project compositeViewsPlugin % "compile->compile", elasticsearchPlugin % "compile->compile", storagePlugin % "compile->compile;test->test", - searchPlugin % "compile->compile", tests % "test->compile;test->test" ) .settings( diff --git a/ship/src/main/resources/ship-default.conf b/ship/src/main/resources/ship-default.conf index a9a850aa89..d233502cf9 100644 --- a/ship/src/main/resources/ship-default.conf +++ b/ship/src/main/resources/ship-default.conf @@ -64,11 +64,6 @@ ship { name = "Default Sparql view" description = "A Sparql view of all resources in the project." } - - search { - name = "Default global search view" - description = "An Elasticsearch view of configured resources for the global search." - } } # Service account configuration for internal operations diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ViewDefaults.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ViewDefaults.scala index c2f7b4ae45..6c638c57a3 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ViewDefaults.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ViewDefaults.scala @@ -6,8 +6,7 @@ import pureconfig.generic.semiauto.deriveReader case class ViewDefaults( elasticsearch: Defaults, - blazegraph: Defaults, - search: Defaults + blazegraph: Defaults ) object ViewDefaults { diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/CompositeViewProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/CompositeViewProcessor.scala index 90c6c5b49f..6508ee2807 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/CompositeViewProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/CompositeViewProcessor.scala @@ -6,7 +6,6 @@ import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.CompositeViews import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewEvent import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewEvent._ import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewRejection.{IncorrectRev, ResourceAlreadyExists} -import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext @@ -38,31 +37,13 @@ class CompositeViewProcessor(views: UUID => IO[CompositeViews], projectMapper: P implicit val c: Caller = Caller(s, Set.empty) val cRev = event.rev - 1 val project = projectMapper.map(event.project) - val searchViewId = nxv + "searchView" event match { - case e: CompositeViewCreated => - e.id match { - case id if id == searchViewId => IO.unit // The search view is created upon project creation - case _ => views(event.uuid).flatMap(_.create(project, e.source)) - } - case e: CompositeViewUpdated => - e.id match { - case id if id == searchViewId => IO.unit - case _ => views(event.uuid).flatMap(_.update(e.id, project, cRev, e.source)) - } - case e: CompositeViewDeprecated => - e.id match { - case id if id == searchViewId => IO.unit - case _ => views(event.uuid).flatMap(_.deprecate(e.id, project, cRev)) - } - case e: CompositeViewUndeprecated => - e.id match { - case id if id == searchViewId => IO.unit - case _ => views(event.uuid).flatMap(_.undeprecate(e.id, project, cRev)) - } - case _: CompositeViewTagAdded => - IO.unit // TODO: Can/should we tag? + case e: CompositeViewCreated => views(event.uuid).flatMap(_.create(project, e.source)) + case e: CompositeViewUpdated => views(event.uuid).flatMap(_.update(e.id, project, cRev, e.source)) + case e: CompositeViewDeprecated => views(event.uuid).flatMap(_.deprecate(e.id, project, cRev)) + case e: CompositeViewUndeprecated => views(event.uuid).flatMap(_.undeprecate(e.id, project, cRev)) + case _: CompositeViewTagAdded => IO.unit // TODO: Can/should we tag? } }.redeemWith( { diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala index c38a82359e..ea5a2142cc 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala @@ -11,7 +11,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.resources.Resources import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors import ch.epfl.bluebrain.nexus.delta.sourcing.model.EntityType import ch.epfl.bluebrain.nexus.delta.sourcing.offset.Offset -import ch.epfl.bluebrain.nexus.delta.sourcing.postgres.Doobie.{PostgresPassword, PostgresUser, transactors} +import ch.epfl.bluebrain.nexus.delta.sourcing.postgres.Doobie.{transactors, PostgresPassword, PostgresUser} import ch.epfl.bluebrain.nexus.ship.ImportReport.Count import ch.epfl.bluebrain.nexus.ship.RunShipSuite.{checkFor, clearDB, expectedImportReport, getDistinctOrgProjects} import ch.epfl.bluebrain.nexus.testkit.config.SystemPropertyOverride From 522b49aca95a933b2e4f3a042a28ddff14db0166 Mon Sep 17 00:00:00 2001 From: Oliver <20188437+olivergrabinski@users.noreply.github.com> Date: Mon, 15 Apr 2024 15:17:25 +0200 Subject: [PATCH 6/8] Deal with UUIDs --- .../epfl/bluebrain/nexus/ship/RunShip.scala | 2 +- .../ship/projects/ProjectProcessor.scala | 11 +-- .../ship/views/BlazegraphViewProcessor.scala | 3 +- .../views/ElasticSearchViewProcessor.scala | 9 +-- .../nexus/ship/views/ViewWiring.scala | 68 ++++++++++--------- .../nexus/ship/config/ShipConfigSuite.scala | 11 ++- 6 files changed, 51 insertions(+), 53 deletions(-) diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala index 6c81b3ed74..34038bab37 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala @@ -66,7 +66,7 @@ trait RunShip { resolverProcessor = ResolverProcessor(fetchContext, projectMapper, eventLogConfig, eventClock, xas) schemaProcessor = SchemaProcessor(schemaLog, fetchContext, schemaImports, rcr, projectMapper, eventClock) resourceProcessor = ResourceProcessor(resourceLog, rcr, projectMapper, fetchContext, eventClock) - esViewsProcessor <- ElasticSearchViewProcessor(fetchContext, rcr, projectMapper, eventLogConfig, eventClock, xas) + esViewsProcessor = ElasticSearchViewProcessor(fetchContext, rcr, projectMapper, eventLogConfig, eventClock, xas) bgViewsProcessor = BlazegraphViewProcessor(fetchContext, rcr, projectMapper, eventLogConfig, eventClock, xas) compositeViewsProcessor = CompositeViewProcessor(fetchContext, rcr, projectMapper, eventLogConfig, eventClock, xas) // format: on diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala index b716b4b418..3ba427f2df 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala @@ -2,7 +2,6 @@ package ch.epfl.bluebrain.nexus.ship.projects import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.Logger -import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi import ch.epfl.bluebrain.nexus.delta.sdk.ScopeInitializer import ch.epfl.bluebrain.nexus.delta.sdk.model.BaseUri @@ -15,11 +14,11 @@ import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject import ch.epfl.bluebrain.nexus.delta.sourcing.model.{EntityType, ProjectRef} +import ch.epfl.bluebrain.nexus.ship._ import ch.epfl.bluebrain.nexus.ship.config.ShipConfig import ch.epfl.bluebrain.nexus.ship.error.ShipError.ProjectDeletionIsNotAllowed import ch.epfl.bluebrain.nexus.ship.projects.ProjectProcessor.logger import ch.epfl.bluebrain.nexus.ship.views.ViewWiring -import ch.epfl.bluebrain.nexus.ship._ import io.circe.Decoder final class ProjectProcessor private ( @@ -87,12 +86,8 @@ object ProjectProcessor { jsonLdApi: JsonLdApi ): IO[ProjectProcessor] = for { - uuidF <- EventUUIDF.init() - esViewUuid <- UUIDF.random() - bgViewUuid <- UUIDF.random() - esViews <- ViewWiring.esViews(fetchContext, rcr, config.eventLog, clock, xas).flatMap(_(esViewUuid)) - bgViews <- ViewWiring.bgViews(fetchContext, rcr, config.eventLog, clock, xas)(jsonLdApi)(bgViewUuid) - initializer = ViewWiring.viewInitializer(esViews, bgViews, config) + uuidF <- EventUUIDF.init() + initializer <- ViewWiring.viewInitializer2(fetchContext, rcr, config, clock, xas) } yield { val disableDeletion: ValidateProjectDeletion = (p: ProjectRef) => IO.raiseError(ProjectDeletionIsNotAllowed(p)) val projects = ProjectsImpl( diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/BlazegraphViewProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/BlazegraphViewProcessor.scala index d814fa4268..cd0d432a14 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/BlazegraphViewProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/BlazegraphViewProcessor.scala @@ -2,6 +2,7 @@ package ch.epfl.bluebrain.nexus.ship.views import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.Logger +import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.BlazegraphViews import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.BlazegraphViewEvent._ import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.BlazegraphViewRejection.{IncorrectRev, ResourceAlreadyExists} @@ -88,7 +89,7 @@ object BlazegraphViewProcessor { )(implicit jsonLdApi: JsonLdApi ): BlazegraphViewProcessor = { - val views = ViewWiring.bgViews(fetchContext, rcr, config, clock, xas) + val views = (uuid: UUID) => ViewWiring.blazegraphViews(fetchContext, rcr, config, clock, UUIDF.fixed(uuid), xas) new BlazegraphViewProcessor(views, projectMapper, clock) } diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ElasticSearchViewProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ElasticSearchViewProcessor.scala index fcac6098c1..14d1b6be6c 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ElasticSearchViewProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ElasticSearchViewProcessor.scala @@ -2,6 +2,7 @@ package ch.epfl.bluebrain.nexus.ship.views import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.Logger +import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.ElasticSearchViews import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.ElasticSearchViewEvent._ import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.ElasticSearchViewRejection.{IncorrectRev, ResourceAlreadyExists} @@ -88,9 +89,9 @@ object ElasticSearchViewProcessor { xas: Transactors )(implicit jsonLdApi: JsonLdApi - ): IO[ElasticSearchViewProcessor] = - ViewWiring.esViews(fetchContext, rcr, config, clock, xas).map { views => - new ElasticSearchViewProcessor(views, projectMapper, clock) - } + ): ElasticSearchViewProcessor = { + val views = (uuid: UUID) => ViewWiring.elasticSearchViews(fetchContext, rcr, config, clock, UUIDF.fixed(uuid), xas) + new ElasticSearchViewProcessor(views, projectMapper, clock) + } } diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ViewWiring.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ViewWiring.scala index d87474e6c4..43942792b5 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ViewWiring.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ViewWiring.scala @@ -9,14 +9,12 @@ import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.{CompositeViews, Val import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.{ElasticSearchFiles, ElasticSearchViewValue} import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.{ElasticSearchScopeInitialization, ElasticSearchViews, ValidateElasticSearchView} import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi -import ch.epfl.bluebrain.nexus.delta.sdk.error.ServiceError import ch.epfl.bluebrain.nexus.delta.sdk.projects.{FetchContext, ScopeInitializationErrorStore} import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution import ch.epfl.bluebrain.nexus.delta.sdk.views.IndexingRev import ch.epfl.bluebrain.nexus.delta.sdk.{ScopeInitialization, ScopeInitializer} import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig -import ch.epfl.bluebrain.nexus.delta.sourcing.model.{EntityType, ProjectRef} import ch.epfl.bluebrain.nexus.ship.EventClock import ch.epfl.bluebrain.nexus.ship.config.ShipConfig @@ -25,11 +23,12 @@ import scala.concurrent.duration.DurationInt object ViewWiring { - def esViews( + def elasticSearchViews( fetchContext: FetchContext, rcr: ResolverContextResolution, config: EventLogConfig, clock: EventClock, + uuidF: UUIDF, xas: Transactors )(implicit jsonLdApi: JsonLdApi) = { implicit val loader: ClasspathResourceLoader = ClasspathResourceLoader.withContext(getClass) @@ -39,7 +38,7 @@ object ViewWiring { override def apply(uuid: UUID, indexingRev: IndexingRev, v: ElasticSearchViewValue): IO[Unit] = IO.unit } - ElasticSearchFiles.mk(loader).map { files => (uuid: UUID) => + ElasticSearchFiles.mk(loader).flatMap { files => ElasticSearchViews( fetchContext, rcr, @@ -50,32 +49,32 @@ object ViewWiring { files.defaultMapping, files.defaultSettings, clock - )(jsonLdApi, UUIDF.fixed(uuid)) + )(jsonLdApi, uuidF) } } - def bgViews( + def blazegraphViews( fetchContext: FetchContext, rcr: ResolverContextResolution, config: EventLogConfig, clock: EventClock, + uuidF: UUIDF, xas: Transactors )(implicit jsonLdApi: JsonLdApi) = { val noValidation = new ValidateBlazegraphView { override def apply(value: BlazegraphViewValue): IO[Unit] = IO.unit } val prefix = "nexus" // TODO: use the config? - (uuid: UUID) => - BlazegraphViews( - fetchContext, - rcr, - noValidation, - (_: ViewResource) => IO.unit, - config, - prefix, - xas, - clock - )(jsonLdApi, UUIDF.fixed(uuid)) + BlazegraphViews( + fetchContext, + rcr, + noValidation, + (_: ViewResource) => IO.unit, + config, + prefix, + xas, + clock + )(jsonLdApi, uuidF) } def cvViews( @@ -100,10 +99,27 @@ object ViewWiring { )(jsonLdApi, UUIDF.fixed(uuid)) } - def viewInitializer( + def viewInitializer2( + fetchContext: FetchContext, + rcr: ResolverContextResolution, + config: ShipConfig, + clock: EventClock, + xas: Transactors + )(implicit jsonLdApi: JsonLdApi) = { + for { + esViews <- elasticSearchViews(fetchContext, rcr, config.eventLog, clock, UUIDF.random, xas) + bgViews <- blazegraphViews(fetchContext, rcr, config.eventLog, clock, UUIDF.random, xas) + } yield { + viewInitializer(esViews, bgViews, config, clock, xas) + } + } + + private def viewInitializer( esViews: ElasticSearchViews, bgViews: BlazegraphViews, - config: ShipConfig + config: ShipConfig, + clock: EventClock, + xas: Transactors ) = { val viewInits = Set.empty[ScopeInitialization] + new ElasticSearchScopeInitialization( @@ -115,19 +131,7 @@ object ViewWiring { config.serviceAccount.value, config.viewDefaults.blazegraph ) - ScopeInitializer(viewInits, noopErrorStore) - } - - def noopErrorStore: ScopeInitializationErrorStore = new ScopeInitializationErrorStore { - override def save( - entityType: EntityType, - project: ProjectRef, - e: ServiceError.ScopeInitializationFailed - ): IO[Unit] = IO.unit - - override def fetch: IO[List[ScopeInitializationErrorStore.ScopeInitErrorRow]] = IO.pure(List.empty) - - override def delete(project: ProjectRef): IO[Unit] = IO.unit + ScopeInitializer(viewInits, ScopeInitializationErrorStore.apply(xas, clock)) } } diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigSuite.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigSuite.scala index a77b1161e2..b89fde37ae 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigSuite.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigSuite.scala @@ -3,7 +3,7 @@ package ch.epfl.bluebrain.nexus.ship.config import ch.epfl.bluebrain.nexus.delta.sdk.Defaults import ch.epfl.bluebrain.nexus.delta.sdk.model.BaseUri import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Label, ProjectRef} -import ch.epfl.bluebrain.nexus.ship.config.ShipConfigSuite.{defaultBgValues, defaultEsValues, defaultSearchValues} +import ch.epfl.bluebrain.nexus.ship.config.ShipConfigSuite.{defaultBgValues, defaultEsValues} import ch.epfl.bluebrain.nexus.testkit.mu.NexusSuite import eu.timepit.refined.types.string.NonEmptyString import fs2.aws.s3.models.Models.BucketName @@ -21,8 +21,7 @@ class ShipConfigSuite extends NexusSuite { test("The defaults (name/description) for views should be correct") { val config = ShipConfig.load(None) config.map(_.viewDefaults.elasticsearch).assertEquals(defaultEsValues) >> - config.map(_.viewDefaults.blazegraph).assertEquals(defaultBgValues) >> - config.map(_.viewDefaults.search).assertEquals(defaultSearchValues) + config.map(_.viewDefaults.blazegraph).assertEquals(defaultBgValues) } test("Default configuration should be overloaded by the external config") { @@ -58,9 +57,7 @@ class ShipConfigSuite extends NexusSuite { } object ShipConfigSuite { - private val defaultEsValues = + private val defaultEsValues = Defaults("Default Elasticsearch view", "An Elasticsearch view of all resources in the project.") - private val defaultBgValues = Defaults("Default Sparql view", "A Sparql view of all resources in the project.") - private val defaultSearchValues = - Defaults("Default global search view", "An Elasticsearch view of configured resources for the global search.") + private val defaultBgValues = Defaults("Default Sparql view", "A Sparql view of all resources in the project.") } From 532227e8151b99d0dc20f1e378a58c0b82cb19cb Mon Sep 17 00:00:00 2001 From: Oliver <20188437+olivergrabinski@users.noreply.github.com> Date: Mon, 15 Apr 2024 15:19:02 +0200 Subject: [PATCH 7/8] Rename method --- .../nexus/ship/projects/ProjectProcessor.scala | 2 +- .../epfl/bluebrain/nexus/ship/views/ViewWiring.scala | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala index 3ba427f2df..4b5c1d896b 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala @@ -87,7 +87,7 @@ object ProjectProcessor { ): IO[ProjectProcessor] = for { uuidF <- EventUUIDF.init() - initializer <- ViewWiring.viewInitializer2(fetchContext, rcr, config, clock, xas) + initializer <- ViewWiring.viewInitializer(fetchContext, rcr, config, clock, xas) } yield { val disableDeletion: ValidateProjectDeletion = (p: ProjectRef) => IO.raiseError(ProjectDeletionIsNotAllowed(p)) val projects = ProjectsImpl( diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ViewWiring.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ViewWiring.scala index 43942792b5..c198b36d3d 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ViewWiring.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ViewWiring.scala @@ -99,19 +99,17 @@ object ViewWiring { )(jsonLdApi, UUIDF.fixed(uuid)) } - def viewInitializer2( + def viewInitializer( fetchContext: FetchContext, rcr: ResolverContextResolution, config: ShipConfig, clock: EventClock, xas: Transactors - )(implicit jsonLdApi: JsonLdApi) = { + )(implicit jsonLdApi: JsonLdApi): IO[ScopeInitializer] = { for { esViews <- elasticSearchViews(fetchContext, rcr, config.eventLog, clock, UUIDF.random, xas) bgViews <- blazegraphViews(fetchContext, rcr, config.eventLog, clock, UUIDF.random, xas) - } yield { - viewInitializer(esViews, bgViews, config, clock, xas) - } + } yield viewInitializer(esViews, bgViews, config, clock, xas) } private def viewInitializer( @@ -120,7 +118,7 @@ object ViewWiring { config: ShipConfig, clock: EventClock, xas: Transactors - ) = { + ): ScopeInitializer = { val viewInits = Set.empty[ScopeInitialization] + new ElasticSearchScopeInitialization( esViews, From 10bb75ccccc048475d36e214b3ec2fbc24ff82e2 Mon Sep 17 00:00:00 2001 From: Oliver <20188437+olivergrabinski@users.noreply.github.com> Date: Mon, 15 Apr 2024 17:20:51 +0200 Subject: [PATCH 8/8] Delete temp file --- ship/src/test/resources/config/views.conf | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 ship/src/test/resources/config/views.conf diff --git a/ship/src/test/resources/config/views.conf b/ship/src/test/resources/config/views.conf deleted file mode 100644 index 2a98111bb3..0000000000 --- a/ship/src/test/resources/config/views.conf +++ /dev/null @@ -1,5 +0,0 @@ -ship { - - - -} \ No newline at end of file