Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

View initialization is nexus ship #4868

Merged
merged 8 commits into from
Apr 15, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions ship/src/main/resources/ship-default.conf
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,18 @@ ship {
}
}

view-defaults {
elasticsearch {
name = "Default Elasticsearch view"
description = "An Elasticsearch view of all resources in the project."
}

blazegraph {
name = "Default Sparql view"
description = "A Sparql view of all resources in the project."
}
}

# Service account configuration for internal operations
service-account {
subject: "delta"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,11 +62,11 @@ trait RunShip {
rcr = ContextWiring.resolverContextResolution(fetchResource, fetchContext, remoteContextResolution, eventLogConfig, eventClock, xas)
schemaImports = SchemaWiring.schemaImports(fetchResource, fetchSchema, fetchContext, eventLogConfig, eventClock, xas)
// Processors
projectProcessor <- ProjectProcessor(fetchActiveOrg, projectMapper, eventLogConfig, eventClock, xas)(baseUri)
projectProcessor <- ProjectProcessor(fetchActiveOrg, fetchContext, rcr, projectMapper, config, eventClock, xas)(baseUri, jsonLdApi)
resolverProcessor = ResolverProcessor(fetchContext, projectMapper, eventLogConfig, eventClock, xas)
schemaProcessor = SchemaProcessor(schemaLog, fetchContext, schemaImports, rcr, projectMapper, eventClock)
resourceProcessor = ResourceProcessor(resourceLog, rcr, projectMapper, fetchContext, eventClock)
esViewsProcessor <- ElasticSearchViewProcessor(fetchContext, rcr, projectMapper, eventLogConfig, eventClock, xas)
esViewsProcessor = ElasticSearchViewProcessor(fetchContext, rcr, projectMapper, eventLogConfig, eventClock, xas)
bgViewsProcessor = BlazegraphViewProcessor(fetchContext, rcr, projectMapper, eventLogConfig, eventClock, xas)
compositeViewsProcessor = CompositeViewProcessor(fetchContext, rcr, projectMapper, eventLogConfig, eventClock, xas)
// format: on
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ final case class ShipConfig(
eventLog: EventLogConfig,
organizations: OrganizationCreationConfig,
projectMapping: ProjectMapping = Map.empty,
viewDefaults: ViewDefaults,
serviceAccount: ServiceAccountConfig
)

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package ch.epfl.bluebrain.nexus.ship.config

import ch.epfl.bluebrain.nexus.delta.sdk.Defaults
import pureconfig.ConfigReader
import pureconfig.generic.semiauto.deriveReader

case class ViewDefaults(
elasticsearch: Defaults,
blazegraph: Defaults
)

object ViewDefaults {
implicit val viewDefaultsConfigReader: ConfigReader[ViewDefaults] =
deriveReader[ViewDefaults]
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,27 +2,31 @@ package ch.epfl.bluebrain.nexus.ship.projects

import cats.effect.IO
import ch.epfl.bluebrain.nexus.delta.kernel.Logger
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi
import ch.epfl.bluebrain.nexus.delta.sdk.ScopeInitializer
import ch.epfl.bluebrain.nexus.delta.sdk.model.BaseUri
import ch.epfl.bluebrain.nexus.delta.sdk.organizations.FetchActiveOrganization
import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ProjectEvent._
import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ProjectRejection.NotFound
import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.{ApiMappings, ProjectEvent, ProjectFields, ProjectRejection}
import ch.epfl.bluebrain.nexus.delta.sdk.projects.{Projects, ProjectsImpl, ValidateProjectDeletion}
import ch.epfl.bluebrain.nexus.delta.sdk.projects.{FetchContext, Projects, ProjectsImpl, ValidateProjectDeletion}
import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution
import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors
import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig
import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject
import ch.epfl.bluebrain.nexus.delta.sourcing.model.{EntityType, ProjectRef}
import ch.epfl.bluebrain.nexus.ship._
import ch.epfl.bluebrain.nexus.ship.config.ShipConfig
import ch.epfl.bluebrain.nexus.ship.error.ShipError.ProjectDeletionIsNotAllowed
import ch.epfl.bluebrain.nexus.ship.projects.ProjectProcessor.logger
import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, EventUUIDF, ImportStatus, ProjectMapper}
import ch.epfl.bluebrain.nexus.ship.views.ViewWiring
import io.circe.Decoder

final class ProjectProcessor private (
projects: Projects,
projectMapper: ProjectMapper,
clock: EventClock,
uuidF: EventUUIDF
uuidF: EventUUIDF,
scopeInitializer: ScopeInitializer
) extends EventProcessor[ProjectEvent] {
override def resourceType: EntityType = Projects.entityType

Expand All @@ -44,7 +48,8 @@ final class ProjectProcessor private (
event match {
case ProjectCreated(_, _, _, _, _, description, apiMappings, base, vocab, enforceSchema, _, _) =>
val fields = ProjectFields(description, apiMappings, Some(base), Some(vocab), enforceSchema)
projects.create(projectRef, fields)
projects.create(projectRef, fields) >>
scopeInitializer.initializeProject(projectRef)
case ProjectUpdated(_, _, _, _, _, description, apiMappings, base, vocab, enforceSchema, _, _) =>
val fields = ProjectFields(description, apiMappings, Some(base), Some(vocab), enforceSchema)
projects.update(projectRef, cRev, fields)
Expand All @@ -70,26 +75,30 @@ object ProjectProcessor {
private val logger = Logger[ProjectProcessor]
def apply(
fetchActiveOrg: FetchActiveOrganization,
fetchContext: FetchContext,
rcr: ResolverContextResolution,
projectMapper: ProjectMapper,
config: EventLogConfig,
config: ShipConfig,
clock: EventClock,
xas: Transactors
)(implicit
base: BaseUri
base: BaseUri,
jsonLdApi: JsonLdApi
): IO[ProjectProcessor] =
for {
uuidF <- EventUUIDF.init()
uuidF <- EventUUIDF.init()
initializer <- ViewWiring.viewInitializer(fetchContext, rcr, config, clock, xas)
} yield {
val disableDeletion: ValidateProjectDeletion = (p: ProjectRef) => IO.raiseError(ProjectDeletionIsNotAllowed(p))
val projects = ProjectsImpl(
fetchActiveOrg,
disableDeletion,
ScopeInitializer.noop,
ApiMappings.empty,
config,
config.eventLog,
xas,
clock
)(base, uuidF)
new ProjectProcessor(projects, projectMapper, clock, uuidF)
new ProjectProcessor(projects, projectMapper, clock, uuidF, initializer)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ package ch.epfl.bluebrain.nexus.ship.views
import cats.effect.IO
import ch.epfl.bluebrain.nexus.delta.kernel.Logger
import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF
import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.BlazegraphViews
import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.BlazegraphViewEvent._
import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.BlazegraphViewRejection.{IncorrectRev, ResourceAlreadyExists}
import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.{defaultViewId, BlazegraphViewEvent, BlazegraphViewValue, ViewResource}
import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.{BlazegraphViews, ValidateBlazegraphView}
import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.{defaultViewId, BlazegraphViewEvent}
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi
import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller
import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext
Expand Down Expand Up @@ -45,16 +45,24 @@ class BlazegraphViewProcessor private (
event match {
case e: BlazegraphViewCreated =>
e.id match {
case id if id == defaultViewId => views(event.uuid).flatMap(_.create(e.id, project, e.value))
case id if id == defaultViewId => IO.unit // the default view is created on project creation
case _ => views(event.uuid).flatMap(_.create(e.id, project, e.source))
}
case e: BlazegraphViewUpdated =>
e.id match {
case id if id == defaultViewId => views(event.uuid).flatMap(_.update(e.id, project, cRev, e.value))
case id if id == defaultViewId => IO.unit
case _ => views(event.uuid).flatMap(_.update(e.id, project, cRev, e.source))
}
case e: BlazegraphViewDeprecated => views(event.uuid).flatMap(_.deprecate(e.id, project, cRev))
case e: BlazegraphViewUndeprecated => views(event.uuid).flatMap(_.undeprecate(e.id, project, cRev))
case e: BlazegraphViewDeprecated =>
e.id match {
case id if id == defaultViewId => IO.unit
case _ => views(event.uuid).flatMap(_.deprecate(e.id, project, cRev))
}
case e: BlazegraphViewUndeprecated =>
e.id match {
case id if id == defaultViewId => IO.unit
case _ => views(event.uuid).flatMap(_.undeprecate(e.id, project, cRev))
}
case _: BlazegraphViewTagAdded => IO.unit // TODO: Can we tag?
}
}.redeemWith(
Expand All @@ -81,22 +89,7 @@ object BlazegraphViewProcessor {
)(implicit
jsonLdApi: JsonLdApi
): BlazegraphViewProcessor = {
val noValidation = new ValidateBlazegraphView {
override def apply(value: BlazegraphViewValue): IO[Unit] = IO.unit
}
val prefix = "nexus" // TODO: use the config?

val views = (uuid: UUID) =>
BlazegraphViews(
fetchContext,
rcr,
noValidation,
(_: ViewResource) => IO.unit,
config,
prefix,
xas,
clock
)(jsonLdApi, UUIDF.fixed(uuid))
val views = (uuid: UUID) => ViewWiring.blazegraphViews(fetchContext, rcr, config, clock, UUIDF.fixed(uuid), xas)
new BlazegraphViewProcessor(views, projectMapper, clock)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,10 @@ package ch.epfl.bluebrain.nexus.ship.views

import cats.effect.IO
import ch.epfl.bluebrain.nexus.delta.kernel.Logger
import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF
import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.CompositeViews
import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewEvent
import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewEvent._
import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.CompositeViewRejection.{IncorrectRev, ResourceAlreadyExists}
import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.{CompositeViewEvent, CompositeViewValue}
import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.{CompositeViews, ValidateCompositeView}
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi
import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller
import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext
Expand All @@ -20,7 +19,6 @@ import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, ImportStatus, P
import io.circe.Decoder

import java.util.UUID
import scala.concurrent.duration.DurationInt

class CompositeViewProcessor(views: UUID => IO[CompositeViews], projectMapper: ProjectMapper, clock: EventClock)
extends EventProcessor[CompositeViewEvent] {
Expand All @@ -39,6 +37,7 @@ class CompositeViewProcessor(views: UUID => IO[CompositeViews], projectMapper: P
implicit val c: Caller = Caller(s, Set.empty)
val cRev = event.rev - 1
val project = projectMapper.map(event.project)

event match {
case e: CompositeViewCreated => views(event.uuid).flatMap(_.create(project, e.source))
case e: CompositeViewUpdated => views(event.uuid).flatMap(_.update(e.id, project, cRev, e.source))
Expand Down Expand Up @@ -70,23 +69,8 @@ object CompositeViewProcessor {
)(implicit
jsonLdApi: JsonLdApi
): CompositeViewProcessor = {
val noValidation = new ValidateCompositeView {
override def apply(uuid: UUID, value: CompositeViewValue): IO[Unit] = IO.unit
}

val views = (uuid: UUID) =>
CompositeViews(
fetchContext,
rcr,
noValidation,
3.seconds,
config,
xas,
clock
)(jsonLdApi, UUIDF.fixed(uuid))

val views = ViewWiring.cvViews(fetchContext, rcr, config, clock, xas)
new CompositeViewProcessor(views, projectMapper, clock)

}

}
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,15 @@ package ch.epfl.bluebrain.nexus.ship.views

import cats.effect.IO
import ch.epfl.bluebrain.nexus.delta.kernel.Logger
import ch.epfl.bluebrain.nexus.delta.kernel.utils.{ClasspathResourceLoader, UUIDF}
import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF
import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.ElasticSearchViews
import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.ElasticSearchViewEvent._
import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.ElasticSearchViewRejection.{IncorrectRev, ResourceAlreadyExists}
import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.{defaultViewId, ElasticSearchFiles, ElasticSearchViewEvent, ElasticSearchViewValue}
import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.{ElasticSearchViews, ValidateElasticSearchView}
import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.{defaultViewId, ElasticSearchViewEvent}
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi
import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller
import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext
import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution
import ch.epfl.bluebrain.nexus.delta.sdk.views.IndexingRev
import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors
import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig
import ch.epfl.bluebrain.nexus.delta.sourcing.model.EntityType
Expand Down Expand Up @@ -46,16 +45,24 @@ class ElasticSearchViewProcessor private (
event match {
case e: ElasticSearchViewCreated =>
e.id match {
case id if id == defaultViewId => views(event.uuid).flatMap(_.create(e.id, project, e.value))
case id if id == defaultViewId => IO.unit // the default view is created on project creation
case _ => views(event.uuid).flatMap(_.create(e.id, project, e.source))
}
case e: ElasticSearchViewUpdated =>
e.id match {
case id if id == defaultViewId => views(event.uuid).flatMap(_.update(e.id, project, cRev, e.value))
case id if id == defaultViewId => IO.unit
case _ => views(event.uuid).flatMap(_.update(e.id, project, cRev, e.source))
}
case e: ElasticSearchViewDeprecated => views(event.uuid).flatMap(_.deprecate(e.id, project, cRev))
case e: ElasticSearchViewUndeprecated => views(event.uuid).flatMap(_.undeprecate(e.id, project, cRev))
case e: ElasticSearchViewDeprecated =>
e.id match {
case id if id == defaultViewId => IO.unit
case _ => views(event.uuid).flatMap(_.deprecate(e.id, project, cRev))
}
case e: ElasticSearchViewUndeprecated =>
e.id match {
case id if id == defaultViewId => IO.unit
case _ => views(event.uuid).flatMap(_.undeprecate(e.id, project, cRev))
}
case _: ElasticSearchViewTagAdded => IO.unit // TODO: Check if this is correct
}
}.redeemWith(
Expand All @@ -82,30 +89,9 @@ object ElasticSearchViewProcessor {
xas: Transactors
)(implicit
jsonLdApi: JsonLdApi
): IO[ElasticSearchViewProcessor] = {
implicit val loader: ClasspathResourceLoader = ClasspathResourceLoader.withContext(getClass)

val noValidation = new ValidateElasticSearchView {
override def apply(uuid: UUID, indexingRev: IndexingRev, v: ElasticSearchViewValue): IO[Unit] = IO.unit
}
val prefix = "nexus" // TODO: use the config?
val esFiles = ElasticSearchFiles.mk(loader)

for {
files <- esFiles
views = (uuid: UUID) =>
ElasticSearchViews(
fetchContext,
rcr,
noValidation,
config,
prefix,
xas,
files.defaultMapping,
files.defaultSettings,
clock
)(jsonLdApi, UUIDF.fixed(uuid))
} yield new ElasticSearchViewProcessor(views, projectMapper, clock)
): ElasticSearchViewProcessor = {
val views = (uuid: UUID) => ViewWiring.elasticSearchViews(fetchContext, rcr, config, clock, UUIDF.fixed(uuid), xas)
new ElasticSearchViewProcessor(views, projectMapper, clock)
}

}
Loading