diff --git a/.github/workflows/ci-delta-ship.yml b/.github/workflows/ci-delta-ship.yml index d91a60f46b..ecf8e997c5 100644 --- a/.github/workflows/ci-delta-ship.yml +++ b/.github/workflows/ci-delta-ship.yml @@ -23,6 +23,8 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Create /tmp/ship folder + run: mkdir -p /tmp/ship - name: Setup JDK uses: actions/setup-java@v4 with: @@ -30,8 +32,18 @@ jobs: java-version: '21' cache: 'sbt' check-latest: true - - name: Unit tests + - name: Clean, build Delta & Storage images run: | sbt -Dsbt.color=always -Dsbt.supershell=false \ clean \ - ship-unit-tests-with-coverage + app/Docker/publishLocal + - name: Start services + run: docker-compose -f tests/docker/docker-compose.yml up -d + - name: Waiting for Delta to start + run: | + URL="http://localhost:8080/v1/version" + curl --connect-timeout 3 --max-time 5 --retry 30 --retry-all-errors --retry-delay 3 --retry-max-time 90 $URL + - name: Unit tests + run: | + sbt -Dsbt.color=always -Dsbt.supershell=false \ + ship-unit-tests diff --git a/build.sbt b/build.sbt index 4d8c74ad47..d40820049a 100755 --- a/build.sbt +++ b/build.sbt @@ -741,7 +741,12 @@ lazy val ship = project ) .enablePlugins(UniversalPlugin, JavaAppPackaging, JavaAgent, DockerPlugin, BuildInfoPlugin) .settings(shared, compilation, servicePackaging, assertJavaVersion, kamonSettings, coverage, release) - .dependsOn(sdk % "compile->compile;test->test", testkit % "test->compile") + .dependsOn( + sdk % "compile->compile;test->test", + blazegraphPlugin % "compile->compile", + elasticsearchPlugin % "compile->compile", + tests % "test->compile;test->test" + ) .settings( libraryDependencies ++= Seq(declineEffect), addCompilerPlugin(betterMonadicFor), diff --git a/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/AclsModule.scala b/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/AclsModule.scala index aa758276e3..c701b9d827 100644 --- a/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/AclsModule.scala +++ b/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/AclsModule.scala @@ -37,7 +37,7 @@ object AclsModule extends ModuleDef { permissions.fetchPermissionSet, AclsImpl.findUnknownRealms(xas), permissions.minimum, - config.acls, + config.acls.eventLog, xas, clock ) diff --git a/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/ResourcesModule.scala b/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/ResourcesModule.scala index c8ad4f5630..ff9084e782 100644 --- a/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/ResourcesModule.scala +++ b/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/ResourcesModule.scala @@ -19,18 +19,23 @@ import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ApiMappings import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverResolution.ResourceResolution import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.{ResolverContextResolution, Resolvers, ResourceResolution} +import ch.epfl.bluebrain.nexus.delta.sdk.resources.Resources.{ResourceDefinition, ResourceLog} +import ch.epfl.bluebrain.nexus.delta.sdk.resources._ import ch.epfl.bluebrain.nexus.delta.sdk.resources.model.{Resource, ResourceEvent} -import ch.epfl.bluebrain.nexus.delta.sdk.resources.{DetectChange, Resources, ResourcesConfig, ResourcesImpl, ValidateResource} -import ch.epfl.bluebrain.nexus.delta.sdk.schemas.Schemas +import ch.epfl.bluebrain.nexus.delta.sdk.schemas.FetchSchema import ch.epfl.bluebrain.nexus.delta.sdk.schemas.model.Schema import ch.epfl.bluebrain.nexus.delta.sdk.sse.SseEncoder -import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors +import ch.epfl.bluebrain.nexus.delta.sourcing.{ScopedEventLog, Transactors} import izumi.distage.model.definition.{Id, ModuleDef} /** * Resources wiring */ object ResourcesModule extends ModuleDef { + make[ResourceResolution[Schema]].from { (aclCheck: AclCheck, resolvers: Resolvers, fetchSchema: FetchSchema) => + ResourceResolution.schemaResource(aclCheck, resolvers, fetchSchema, excludeDeprecated = false) + } + make[ValidateResource].from { (resourceResolution: ResourceResolution[Schema], rcr: RemoteContextResolution @Id("aggregate")) => ValidateResource(resourceResolution)(rcr) @@ -40,26 +45,30 @@ object ResourcesModule extends ModuleDef { make[DetectChange].from { (config: ResourcesConfig) => DetectChange(config.skipUpdateNoChange) } + make[ResourceDefinition].from { (validateResource: ValidateResource, detectChange: DetectChange, clock: Clock[IO]) => + Resources.definition(validateResource, detectChange, clock) + } + + make[ResourceLog].from { (scopedDefinition: ResourceDefinition, config: ResourcesConfig, xas: Transactors) => + ScopedEventLog(scopedDefinition, config.eventLog, xas) + } + + make[FetchResource].from { (scopedLog: ResourceLog) => + FetchResource(scopedLog) + } + make[Resources].from { ( - validate: ValidateResource, - detectChange: DetectChange, + resourceLog: ResourceLog, fetchContext: FetchContext, - config: ResourcesConfig, resolverContextResolution: ResolverContextResolution, api: JsonLdApi, - xas: Transactors, - clock: Clock[IO], uuidF: UUIDF ) => ResourcesImpl( - validate, - detectChange, + resourceLog, fetchContext, - resolverContextResolution, - config, - xas, - clock + resolverContextResolution )( api, uuidF @@ -67,12 +76,13 @@ object ResourcesModule extends ModuleDef { } make[ResolverContextResolution].from { - (aclCheck: AclCheck, resolvers: Resolvers, resources: Resources, rcr: RemoteContextResolution @Id("aggregate")) => - ResolverContextResolution(aclCheck, resolvers, resources, rcr) - } - - make[ResourceResolution[Schema]].from { (aclCheck: AclCheck, resolvers: Resolvers, schemas: Schemas) => - ResourceResolution.schemaResource(aclCheck, resolvers, schemas, excludeDeprecated = false) + ( + aclCheck: AclCheck, + resolvers: Resolvers, + rcr: RemoteContextResolution @Id("aggregate"), + fetchResource: FetchResource + ) => + ResolverContextResolution(aclCheck, resolvers, rcr, fetchResource) } make[ResourcesRoutes].from { diff --git a/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/SchemasModule.scala b/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/SchemasModule.scala index 8cff18a434..e6ebc102e4 100644 --- a/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/SchemasModule.scala +++ b/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/SchemasModule.scala @@ -21,11 +21,12 @@ import ch.epfl.bluebrain.nexus.delta.sdk.model.metrics.ScopedEventMetricEncoder import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ApiMappings import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.{ResolverContextResolution, Resolvers} -import ch.epfl.bluebrain.nexus.delta.sdk.resources.Resources +import ch.epfl.bluebrain.nexus.delta.sdk.resources.FetchResource +import ch.epfl.bluebrain.nexus.delta.sdk.schemas.Schemas.{SchemaDefinition, SchemaLog} +import ch.epfl.bluebrain.nexus.delta.sdk.schemas._ import ch.epfl.bluebrain.nexus.delta.sdk.schemas.model.{Schema, SchemaEvent} -import ch.epfl.bluebrain.nexus.delta.sdk.schemas.{SchemaImports, Schemas, SchemasImpl, ValidateSchema} import ch.epfl.bluebrain.nexus.delta.sdk.sse.SseEncoder -import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors +import ch.epfl.bluebrain.nexus.delta.sourcing.{ScopedEventLog, Transactors} import izumi.distage.model.definition.{Id, ModuleDef} /** @@ -40,26 +41,32 @@ object SchemasModule extends ModuleDef { } + make[SchemaDefinition].from { (validateSchema: ValidateSchema, clock: Clock[IO]) => + Schemas.definition(validateSchema, clock) + } + + make[SchemaLog].from { (scopedDefinition: SchemaDefinition, config: AppConfig, xas: Transactors) => + ScopedEventLog(scopedDefinition, config.schemas.eventLog, xas) + } + + make[FetchSchema].from { (schemaLog: SchemaLog) => + FetchSchema(schemaLog) + } + make[Schemas].from { ( + schemaLog: SchemaLog, fetchContext: FetchContext, schemaImports: SchemaImports, api: JsonLdApi, - validate: ValidateSchema, resolverContextResolution: ResolverContextResolution, - config: AppConfig, - xas: Transactors, - clock: Clock[IO], uuidF: UUIDF ) => SchemasImpl( + schemaLog, fetchContext, schemaImports, - resolverContextResolution, - validate, - config.schemas, - xas, - clock + resolverContextResolution )(api, uuidF) } @@ -67,10 +74,10 @@ object SchemasModule extends ModuleDef { ( aclCheck: AclCheck, resolvers: Resolvers, - resources: Resources, - schemas: Schemas + fetchSchema: FetchSchema, + fetchResource: FetchResource ) => - SchemaImports(aclCheck, resolvers, schemas, resources) + SchemaImports(aclCheck, resolvers, fetchSchema, fetchResource) } make[SchemasRoutes].from { diff --git a/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/MainSuite.scala b/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/MainSuite.scala index c1b35b6513..7aadcbac6b 100644 --- a/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/MainSuite.scala +++ b/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/MainSuite.scala @@ -17,6 +17,7 @@ import munit.catseffect.IOFixture import munit.{AnyFixture, CatsEffectSuite} import java.nio.file.{Files, Paths} +import scala.concurrent.duration.Duration /** * Test class that allows to check that across core and plugins: @@ -26,6 +27,8 @@ import java.nio.file.{Files, Paths} */ class MainSuite extends NexusSuite with MainSuite.Fixture { + override val munitIOTimeout: Duration = Duration(60, "s") + private val pluginsParentPath = Paths.get("target/plugins").toAbsolutePath private val pluginLoaderConfig = PluginLoaderConfig(pluginsParentPath.toString) diff --git a/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/AclsRoutesSpec.scala b/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/AclsRoutesSpec.scala index a4e6180462..e8c31aed4a 100644 --- a/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/AclsRoutesSpec.scala +++ b/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/AclsRoutesSpec.scala @@ -7,7 +7,7 @@ import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords import ch.epfl.bluebrain.nexus.delta.sdk.acls.model.AclAddress.{Organization, Project, Root} import ch.epfl.bluebrain.nexus.delta.sdk.acls.model.{Acl, AclAddress} -import ch.epfl.bluebrain.nexus.delta.sdk.acls.{AclCheck, Acls, AclsConfig, AclsImpl} +import ch.epfl.bluebrain.nexus.delta.sdk.acls.{AclCheck, Acls, AclsImpl} import ch.epfl.bluebrain.nexus.delta.sdk.identities.IdentitiesDummy import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.implicits._ @@ -90,7 +90,7 @@ class AclsRoutesSpec extends BaseRouteSpec { IO.pure(Set(aclsPermissions.read, aclsPermissions.write, managePermission, events.read)), Acls.findUnknownRealms(_, Set(realm1, realm2)), Set(aclsPermissions.read, aclsPermissions.write, managePermission, events.read), - AclsConfig(EventLogConfig(QueryConfig(5, RefreshStrategy.Stop), 100.millis)), + EventLogConfig(QueryConfig(5, RefreshStrategy.Stop), 100.millis), xas, clock ) diff --git a/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/ResourcesRoutesSpec.scala b/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/ResourcesRoutesSpec.scala index 1a34cd9b27..a0cf8286f1 100644 --- a/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/ResourcesRoutesSpec.scala +++ b/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/ResourcesRoutesSpec.scala @@ -17,16 +17,17 @@ import ch.epfl.bluebrain.nexus.delta.sdk.generators.{ProjectGen, ResourceResolut import ch.epfl.bluebrain.nexus.delta.sdk.identities.IdentitiesDummy import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.implicits._ +import ch.epfl.bluebrain.nexus.delta.sdk.model.Fetch.FetchF import ch.epfl.bluebrain.nexus.delta.sdk.model.{IdSegmentRef, ResourceUris} import ch.epfl.bluebrain.nexus.delta.sdk.permissions.Permissions.resources import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContextDummy import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ApiMappings import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution -import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverResolution.FetchResource import ch.epfl.bluebrain.nexus.delta.sdk.resources.NexusSource.DecodingOption -import ch.epfl.bluebrain.nexus.delta.sdk.resources.{DetectChange, Resources, ResourcesConfig, ResourcesImpl, ValidateResource} +import ch.epfl.bluebrain.nexus.delta.sdk.resources._ import ch.epfl.bluebrain.nexus.delta.sdk.schemas.model.Schema import ch.epfl.bluebrain.nexus.delta.sdk.utils.BaseRouteSpec +import ch.epfl.bluebrain.nexus.delta.sourcing.ScopedEventLog import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.{Anonymous, Authenticated, Group, Subject, User} import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef} @@ -87,7 +88,7 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues { private val aclCheck = AclSimpleCheck().accepted - private val fetchSchema: (ResourceRef, ProjectRef) => FetchResource[Schema] = { + private val fetchSchema: (ResourceRef, ProjectRef) => FetchF[Schema] = { case (ref, _) if ref.iri == schema2.id => IO.pure(Some(SchemaGen.resourceFor(schema2, deprecated = true))) case (ref, _) if ref.iri == schema1.id => IO.pure(Some(SchemaGen.resourceFor(schema1))) case (ref, _) if ref.iri == schema3.id => IO.pure(Some(SchemaGen.resourceFor(schema3))) @@ -101,14 +102,17 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues { private val resolverContextResolution: ResolverContextResolution = ResolverContextResolution(rcr) private def routesWithDecodingOption(implicit decodingOption: DecodingOption): (Route, Resources) = { + val resourceDef = Resources.definition(validator, DetectChange(enabled = true), clock) + val scopedLog = ScopedEventLog( + resourceDef, + ResourcesConfig(eventLogConfig, decodingOption, skipUpdateNoChange = true).eventLog, + xas + ) + val resources = ResourcesImpl( - validator, - DetectChange(enabled = true), + scopedLog, fetchContext, - resolverContextResolution, - ResourcesConfig(eventLogConfig, decodingOption, skipUpdateNoChange = true), - xas, - clock + resolverContextResolution ) ( Route.seal( diff --git a/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/SchemasRoutesSpec.scala b/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/SchemasRoutesSpec.scala index 472d6c3c0e..1d803ba63b 100644 --- a/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/SchemasRoutesSpec.scala +++ b/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/SchemasRoutesSpec.scala @@ -23,8 +23,9 @@ import ch.epfl.bluebrain.nexus.delta.sdk.permissions.Permissions.schemas import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContextDummy import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ApiMappings import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution -import ch.epfl.bluebrain.nexus.delta.sdk.schemas.{SchemaImports, SchemasConfig, SchemasImpl, ValidateSchema} +import ch.epfl.bluebrain.nexus.delta.sdk.schemas.{SchemaImports, Schemas, SchemasConfig, SchemasImpl, ValidateSchema} import ch.epfl.bluebrain.nexus.delta.sdk.utils.BaseRouteSpec +import ch.epfl.bluebrain.nexus.delta.sourcing.ScopedEventLog import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.{Anonymous, Authenticated, Group, Subject, User} import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef import ch.epfl.bluebrain.nexus.testkit.ce.IOFromMap @@ -78,12 +79,15 @@ class SchemasRoutesSpec extends BaseRouteSpec with IOFromMap with CatsIOValues { private val config = SchemasConfig(eventLogConfig) + private val schemaDef = Schemas.definition(ValidateSchema.apply, clock) + private lazy val schemaLog = ScopedEventLog(schemaDef, config.eventLog, xas) + private lazy val routes = Route.seal( SchemasRoutes( identities, aclCheck, - SchemasImpl(fetchContext, schemaImports, resolverContextResolution, ValidateSchema.apply, config, xas, clock), + SchemasImpl(schemaLog, fetchContext, schemaImports, resolverContextResolution), groupDirectives, IndexingAction.noop ) diff --git a/delta/plugins/blazegraph/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/blazegraph/BlazegraphViews.scala b/delta/plugins/blazegraph/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/blazegraph/BlazegraphViews.scala index b28b2bbeda..bc9935b467 100644 --- a/delta/plugins/blazegraph/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/blazegraph/BlazegraphViews.scala +++ b/delta/plugins/blazegraph/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/blazegraph/BlazegraphViews.scala @@ -532,7 +532,7 @@ object BlazegraphViews { apply(fetchContext, contextResolution, validate, createNameSpace, eventLogConfig, prefix, xas, clock) } - private[blazegraph] def apply( + def apply( fetchContext: FetchContext, contextResolution: ResolverContextResolution, validate: ValidateBlazegraphView, diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/acls/AclsImpl.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/acls/AclsImpl.scala index 341279f4b0..306a34cc20 100644 --- a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/acls/AclsImpl.scala +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/acls/AclsImpl.scala @@ -13,6 +13,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.permissions.model.Permission import ch.epfl.bluebrain.nexus.delta.sdk.realms.Realms import ch.epfl.bluebrain.nexus.delta.sdk.syntax._ import ch.epfl.bluebrain.nexus.delta.sourcing._ +import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label import ch.epfl.bluebrain.nexus.delta.sourcing.state.GlobalStateStore @@ -108,12 +109,12 @@ object AclsImpl { fetchPermissionSet: IO[Set[Permission]], findUnknownRealms: Set[Label] => IO[Unit], minimum: Set[Permission], - config: AclsConfig, + config: EventLogConfig, xas: Transactors, clock: Clock[IO] ): Acls = new AclsImpl( - GlobalEventLog(Acls.definition(fetchPermissionSet, findUnknownRealms, clock), config.eventLog, xas), + GlobalEventLog(Acls.definition(fetchPermissionSet, findUnknownRealms, clock), config, xas), minimum ) diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/Fetch.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/Fetch.scala new file mode 100644 index 0000000000..97affef962 --- /dev/null +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/Fetch.scala @@ -0,0 +1,10 @@ +package ch.epfl.bluebrain.nexus.delta.sdk.model + +import cats.effect.IO + +object Fetch { + + type Fetch[R] = IO[Option[R]] + type FetchF[R] = Fetch[ResourceF[R]] + +} diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resolvers/ResolverContextResolution.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resolvers/ResolverContextResolution.scala index c14a36c5a4..574edc2f8d 100644 --- a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resolvers/ResolverContextResolution.scala +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resolvers/ResolverContextResolution.scala @@ -11,7 +11,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution.{logger, ProjectRemoteContext} import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverResolution.ResourceResolution import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.model.ResourceResolutionReport -import ch.epfl.bluebrain.nexus.delta.sdk.resources.Resources +import ch.epfl.bluebrain.nexus.delta.sdk.resources.FetchResource import ch.epfl.bluebrain.nexus.delta.sdk.resources.model.Resource import ch.epfl.bluebrain.nexus.delta.sdk.syntax._ import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef} @@ -111,18 +111,18 @@ object ResolverContextResolution { * how to check acls * @param resolvers * a resolvers instance - * @param resources - * a resource instance * @param rcr * a previously defined 'RemoteContextResolution' + * @param fetchResource + * how to fetch a resource */ def apply( aclCheck: AclCheck, resolvers: Resolvers, - resources: Resources, - rcr: RemoteContextResolution + rcr: RemoteContextResolution, + fetchResource: FetchResource ): ResolverContextResolution = - apply(rcr, ResourceResolution.dataResource(aclCheck, resolvers, resources, excludeDeprecated = false)) + apply(rcr, ResourceResolution.dataResource(aclCheck, resolvers, fetchResource, excludeDeprecated = false)) /** * A [[ResolverContextResolution]] that never resolves diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resolvers/ResolverResolution.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resolvers/ResolverResolution.scala index 1ca104be73..47a24db77a 100644 --- a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resolvers/ResolverResolution.scala +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resolvers/ResolverResolution.scala @@ -2,18 +2,18 @@ package ch.epfl.bluebrain.nexus.delta.sdk.resolvers import cats.effect.IO import cats.implicits._ - import ch.epfl.bluebrain.nexus.delta.kernel.search.Pagination import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.jsonld.JsonLdContent +import ch.epfl.bluebrain.nexus.delta.sdk.model.Fetch.Fetch import ch.epfl.bluebrain.nexus.delta.sdk.model.ResourceF import ch.epfl.bluebrain.nexus.delta.sdk.model.search.ResultEntry import ch.epfl.bluebrain.nexus.delta.sdk.model.search.SearchParams.ResolverSearchParams import ch.epfl.bluebrain.nexus.delta.sdk.permissions.Permissions import ch.epfl.bluebrain.nexus.delta.sdk.permissions.model.Permission -import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverResolution.{DeprecationCheck, Fetch, ResolverResolutionResult} +import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverResolution.{DeprecationCheck, ResolverResolutionResult} import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.model.IdentityResolution.{ProvidedIdentities, UseCurrentCaller} import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.model.Resolver.{CrossProjectResolver, InProjectResolver} import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.model.ResolverResolutionRejection._ @@ -227,10 +227,6 @@ object ResolverResolution { */ type ResourceResolution[R] = ResolverResolution[ResourceF[R]] - type Fetch[R] = IO[Option[R]] - - type FetchResource[R] = IO[Option[ResourceF[R]]] - type ResolverResolutionResult[R] = (ResolverReport, Option[R]) private val resolverSearchParams = ResolverSearchParams(deprecated = Some(false), filter = _ => IO.pure(true)) diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resolvers/ResourceResolution.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resolvers/ResourceResolution.scala index a225041e0d..0f350c51b0 100644 --- a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resolvers/ResourceResolution.scala +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resolvers/ResourceResolution.scala @@ -3,14 +3,15 @@ package ch.epfl.bluebrain.nexus.delta.sdk.resolvers import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck +import ch.epfl.bluebrain.nexus.delta.sdk.model.Fetch.FetchF import ch.epfl.bluebrain.nexus.delta.sdk.model.ResourceF import ch.epfl.bluebrain.nexus.delta.sdk.permissions.Permissions import ch.epfl.bluebrain.nexus.delta.sdk.permissions.model.Permission -import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverResolution.{DeprecationCheck, FetchResource, ResourceResolution} +import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverResolution.{DeprecationCheck, ResourceResolution} import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.model.Resolver -import ch.epfl.bluebrain.nexus.delta.sdk.resources.Resources +import ch.epfl.bluebrain.nexus.delta.sdk.resources.FetchResource import ch.epfl.bluebrain.nexus.delta.sdk.resources.model.Resource -import ch.epfl.bluebrain.nexus.delta.sdk.schemas.Schemas +import ch.epfl.bluebrain.nexus.delta.sdk.schemas.FetchSchema import ch.epfl.bluebrain.nexus.delta.sdk.schemas.model.Schema import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Identity, ProjectRef, ResourceRef} @@ -31,7 +32,7 @@ object ResourceResolution { checkAcls: (ProjectRef, Set[Identity]) => IO[Boolean], listResolvers: ProjectRef => IO[List[Resolver]], fetchResolver: (Iri, ProjectRef) => IO[Resolver], - fetch: (ResourceRef, ProjectRef) => FetchResource[R], + fetch: (ResourceRef, ProjectRef) => FetchF[R], excludeDeprecated: Boolean ): ResourceResolution[R] = new ResolverResolution( @@ -60,7 +61,7 @@ object ResourceResolution { def apply[R]( aclCheck: AclCheck, resolvers: Resolvers, - fetchResource: (ResourceRef, ProjectRef) => FetchResource[R], + fetchResource: (ResourceRef, ProjectRef) => FetchF[R], readPermission: Permission, excludeDeprecated: Boolean ): ResourceResolution[R] = @@ -73,21 +74,21 @@ object ResourceResolution { * how to check acls * @param resolvers * a resolvers instance - * @param resources - * a resources instance + * @param fetchResource + * how to fetch a resource * @param excludeDeprecated * to exclude deprecated resources from the resolution */ def dataResource( aclCheck: AclCheck, resolvers: Resolvers, - resources: Resources, + fetchResource: FetchResource, excludeDeprecated: Boolean ): ResourceResolution[Resource] = apply( aclCheck, resolvers, - (ref: ResourceRef, project: ProjectRef) => resources.fetch(ref, project).redeem(_ => None, Some(_)), + fetchResource.fetch _, Permissions.resources.read, excludeDeprecated ) @@ -99,21 +100,21 @@ object ResourceResolution { * how to check acls * @param resolvers * a resolvers instance - * @param schemas - * a schemas instance + * @param fetchSchema + * how to fetch a schema * @param excludeDeprecated * to exclude deprecated resources from the resolution */ def schemaResource( aclCheck: AclCheck, resolvers: Resolvers, - schemas: Schemas, + fetchSchema: FetchSchema, excludeDeprecated: Boolean ): ResourceResolution[Schema] = apply( aclCheck, resolvers, - (ref: ResourceRef, project: ProjectRef) => schemas.fetch(ref, project).redeem(_ => None, Some(_)), + fetchSchema.fetch _, Permissions.schemas.read, excludeDeprecated ) diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/FetchResource.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/FetchResource.scala new file mode 100644 index 0000000000..0460ca1d3e --- /dev/null +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/FetchResource.scala @@ -0,0 +1,47 @@ +package ch.epfl.bluebrain.nexus.delta.sdk.resources + +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri +import ch.epfl.bluebrain.nexus.delta.sdk.model.Fetch.FetchF +import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegmentRef +import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegmentRef.{Latest, Revision, Tag} +import ch.epfl.bluebrain.nexus.delta.sdk.resources.model.ResourceRejection.{ResourceNotFound, RevisionNotFound, TagNotFound} +import ch.epfl.bluebrain.nexus.delta.sdk.resources.model._ +import ch.epfl.bluebrain.nexus.delta.sourcing.ScopedEventLogReadOnly +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef} + +trait FetchResource { + + /** Fetch the referenced resource in the given project */ + def fetch(ref: ResourceRef, project: ProjectRef): FetchF[Resource] + + def stateOrNotFound(id: IdSegmentRef, iri: Iri, ref: ProjectRef): IO[ResourceState] + +} + +object FetchResource { + + def apply( + log: ScopedEventLogReadOnly[Iri, ResourceState, ResourceRejection] + ): FetchResource = { + + def notFound(iri: Iri, ref: ProjectRef) = ResourceNotFound(iri, ref) + + new FetchResource { + override def fetch(ref: ResourceRef, project: ProjectRef): FetchF[Resource] = { + stateOrNotFound(IdSegmentRef(ref), ref.iri, project).attempt + .map(_.toOption) + .map(_.map(_.toResource)) + } + + override def stateOrNotFound(id: IdSegmentRef, iri: Iri, ref: ProjectRef): IO[ResourceState] = + id match { + case Latest(_) => log.stateOr(ref, iri, notFound(iri, ref)) + case Revision(_, rev) => log.stateOr(ref, iri, rev, notFound(iri, ref), RevisionNotFound) + case Tag(_, tag) => log.stateOr(ref, iri, tag, notFound(iri, ref), TagNotFound(tag)) + } + } + + } + +} diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/Resources.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/Resources.scala index 7d3ea4e277..695a9ba519 100644 --- a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/Resources.scala +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/Resources.scala @@ -19,7 +19,7 @@ import ch.epfl.bluebrain.nexus.delta.sourcing.ScopedEntityDefinition.Tagger import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag import ch.epfl.bluebrain.nexus.delta.sourcing.model._ -import ch.epfl.bluebrain.nexus.delta.sourcing.{ScopedEntityDefinition, StateMachine} +import ch.epfl.bluebrain.nexus.delta.sourcing.{ScopedEntityDefinition, ScopedEventLog, StateMachine} import io.circe.Json /** @@ -505,6 +505,11 @@ object Resources { } } + type ResourceDefinition = + ScopedEntityDefinition[Iri, ResourceState, ResourceCommand, ResourceEvent, ResourceRejection] + type ResourceLog = + ScopedEventLog[Iri, ResourceState, ResourceCommand, ResourceEvent, ResourceRejection] + /** * Entity definition for [[Resources]] */ @@ -512,7 +517,7 @@ object Resources { validateResource: ValidateResource, detectChange: DetectChange, clock: Clock[IO] - ): ScopedEntityDefinition[Iri, ResourceState, ResourceCommand, ResourceEvent, ResourceRejection] = + ): ResourceDefinition = ScopedEntityDefinition( entityType, StateMachine(None, evaluate(validateResource, detectChange, clock)(_, _), next), diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/ResourcesImpl.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/ResourcesImpl.scala index 563a320f6f..d743bea7ee 100644 --- a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/ResourcesImpl.scala +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/ResourcesImpl.scala @@ -1,6 +1,6 @@ package ch.epfl.bluebrain.nexus.delta.sdk.resources -import cats.effect.{Clock, IO} +import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.Logger import ch.epfl.bluebrain.nexus.delta.kernel.kamon.KamonMetricComponent import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF @@ -10,15 +10,14 @@ import ch.epfl.bluebrain.nexus.delta.sdk._ import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.implicits._ import ch.epfl.bluebrain.nexus.delta.sdk.jsonld.JsonLdSourceProcessor.JsonLdSourceResolvingParser -import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegmentRef.{Latest, Revision, Tag} import ch.epfl.bluebrain.nexus.delta.sdk.model._ import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ProjectContext import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution -import ch.epfl.bluebrain.nexus.delta.sdk.resources.Resources.{entityType, expandIri, expandResourceRef} +import ch.epfl.bluebrain.nexus.delta.sdk.resources.Resources.{entityType, expandIri, expandResourceRef, ResourceLog} import ch.epfl.bluebrain.nexus.delta.sdk.resources.ResourcesImpl.{logger, ResourcesLog} import ch.epfl.bluebrain.nexus.delta.sdk.resources.model.ResourceCommand._ -import ch.epfl.bluebrain.nexus.delta.sdk.resources.model.ResourceRejection.{NoChangeDetected, ResourceNotFound, RevisionNotFound, TagNotFound} +import ch.epfl.bluebrain.nexus.delta.sdk.resources.model.ResourceRejection.{NoChangeDetected, ResourceNotFound} import ch.epfl.bluebrain.nexus.delta.sdk.resources.model.{ResourceCommand, ResourceEvent, ResourceRejection, ResourceState} import ch.epfl.bluebrain.nexus.delta.sourcing._ import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject @@ -165,17 +164,11 @@ final class ResourcesImpl private ( for { (iri, pc) <- expandWithContext(fetchContext.onRead, projectRef, id.value) schemaRefOpt <- IO.fromEither(expandResourceRef(schemaOpt, pc)) - state <- stateOrNotFound(id, iri, projectRef) + state <- FetchResource(log).stateOrNotFound(id, iri, projectRef) _ <- IO.raiseWhen(schemaRefOpt.exists(_.iri != state.schema.iri))(notFound(iri, projectRef)) } yield state }.span("fetchResource") - private def stateOrNotFound(id: IdSegmentRef, iri: Iri, ref: ProjectRef): IO[ResourceState] = id match { - case Latest(_) => log.stateOr(ref, iri, notFound(iri, ref)) - case Revision(_, rev) => log.stateOr(ref, iri, rev, notFound(iri, ref), RevisionNotFound) - case Tag(_, tag) => log.stateOr(ref, iri, tag, notFound(iri, ref), TagNotFound(tag)) - } - private def notFound(iri: Iri, ref: ProjectRef) = ResourceNotFound(iri, ref) override def fetch( @@ -210,31 +203,23 @@ object ResourcesImpl { /** * Constructs a [[Resources]] instance. * - * @param validateResource - * how to validate resource + * @param scopedLog + * the scoped resource log * @param fetchContext * to fetch the project context * @param contextResolution * the context resolver - * @param config - * the resources config - * @param xas - * the database context */ final def apply( - validateResource: ValidateResource, - detectChange: DetectChange, + scopedLog: ResourceLog, fetchContext: FetchContext, - contextResolution: ResolverContextResolution, - config: ResourcesConfig, - xas: Transactors, - clock: Clock[IO] + contextResolution: ResolverContextResolution )(implicit api: JsonLdApi, uuidF: UUIDF = UUIDF.random ): Resources = new ResourcesImpl( - ScopedEventLog(Resources.definition(validateResource, detectChange, clock), config.eventLog, xas), + scopedLog, fetchContext, JsonLdSourceResolvingParser(contextResolution, uuidF) ) diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/schemas/FetchSchema.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/schemas/FetchSchema.scala new file mode 100644 index 0000000000..484549f69f --- /dev/null +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/schemas/FetchSchema.scala @@ -0,0 +1,44 @@ +package ch.epfl.bluebrain.nexus.delta.sdk.schemas + +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri +import ch.epfl.bluebrain.nexus.delta.sdk.model.Fetch.FetchF +import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegmentRef +import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegmentRef.{Latest, Revision, Tag} +import ch.epfl.bluebrain.nexus.delta.sdk.schemas.model.SchemaRejection.{RevisionNotFound, SchemaNotFound, TagNotFound} +import ch.epfl.bluebrain.nexus.delta.sdk.schemas.model.{Schema, SchemaRejection, SchemaState} +import ch.epfl.bluebrain.nexus.delta.sourcing.ScopedEventLogReadOnly +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef} + +trait FetchSchema { + + /** Fetch the referenced resource in the given project */ + def fetch(ref: ResourceRef, project: ProjectRef): FetchF[Schema] + + def stateOrNotFound(id: IdSegmentRef, iri: Iri, ref: ProjectRef): IO[SchemaState] + +} + +object FetchSchema { + + def apply(log: ScopedEventLogReadOnly[Iri, SchemaState, SchemaRejection]): FetchSchema = { + + def notFound(iri: Iri, ref: ProjectRef) = SchemaNotFound(iri, ref) + + new FetchSchema { + override def fetch(ref: ResourceRef, project: ProjectRef): FetchF[Schema] = + stateOrNotFound(IdSegmentRef(ref), ref.iri, project).attempt + .map(_.toOption) + .map(_.map(_.toResource)) + + override def stateOrNotFound(id: IdSegmentRef, iri: Iri, ref: ProjectRef): IO[SchemaState] = + id match { + case Latest(_) => log.stateOr(ref, iri, notFound(iri, ref)) + case Revision(_, rev) => log.stateOr(ref, iri, rev, notFound(iri, ref), RevisionNotFound) + case Tag(_, tag) => log.stateOr(ref, iri, tag, notFound(iri, ref), TagNotFound(tag)) + } + } + + } + +} diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/schemas/SchemaImports.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/schemas/SchemaImports.scala index ba887cd476..4a671208ba 100644 --- a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/schemas/SchemaImports.scala +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/schemas/SchemaImports.scala @@ -3,7 +3,6 @@ package ch.epfl.bluebrain.nexus.delta.sdk.schemas import cats.data.NonEmptyList import cats.effect.IO import cats.implicits._ - import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.owl import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.ExpandedJsonLd @@ -12,7 +11,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.model.ResourceResolutionReport import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.{Resolvers, ResourceResolution} -import ch.epfl.bluebrain.nexus.delta.sdk.resources.Resources +import ch.epfl.bluebrain.nexus.delta.sdk.resources.FetchResource import ch.epfl.bluebrain.nexus.delta.sdk.resources.model.Resource import ch.epfl.bluebrain.nexus.delta.sdk.schemas.model.Schema import ch.epfl.bluebrain.nexus.delta.sdk.schemas.model.SchemaRejection.InvalidSchemaResolution @@ -98,19 +97,21 @@ object SchemaImports { final def apply( aclCheck: AclCheck, resolvers: Resolvers, - schemas: Schemas, - resources: Resources + fetchSchema: FetchSchema, + fetchResource: FetchResource ): SchemaImports = { - def resolveSchema(ref: ResourceRef, projectRef: ProjectRef, caller: Caller) = + def resolveSchema(ref: ResourceRef, projectRef: ProjectRef, caller: Caller) = ResourceResolution - .schemaResource(aclCheck, resolvers, schemas, excludeDeprecated = true) + .schemaResource(aclCheck, resolvers, fetchSchema, excludeDeprecated = true) .resolve(ref, projectRef)(caller) .map(_.map(_.value)) + def resolveResource(ref: ResourceRef, projectRef: ProjectRef, caller: Caller) = ResourceResolution - .dataResource(aclCheck, resolvers, resources, excludeDeprecated = true) + .dataResource(aclCheck, resolvers, fetchResource, excludeDeprecated = true) .resolve(ref, projectRef)(caller) .map(_.map(_.value)) + new SchemaImports(resolveSchema, resolveResource) } diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/schemas/Schemas.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/schemas/Schemas.scala index e4f4f8629b..242479d4a6 100644 --- a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/schemas/Schemas.scala +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/schemas/Schemas.scala @@ -19,7 +19,7 @@ import ch.epfl.bluebrain.nexus.delta.sourcing.ScopedEntityDefinition.Tagger import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag import ch.epfl.bluebrain.nexus.delta.sourcing.model._ -import ch.epfl.bluebrain.nexus.delta.sourcing.{ScopedEntityDefinition, StateMachine} +import ch.epfl.bluebrain.nexus.delta.sourcing.{ScopedEntityDefinition, ScopedEventLog, StateMachine} import io.circe.Json /** @@ -375,13 +375,16 @@ object Schemas { } } + type SchemaDefinition = ScopedEntityDefinition[Iri, SchemaState, SchemaCommand, SchemaEvent, SchemaRejection] + type SchemaLog = ScopedEventLog[Iri, SchemaState, SchemaCommand, SchemaEvent, SchemaRejection] + /** * Entity definition for [[Schemas]] */ def definition( validate: ValidateSchema, clock: Clock[IO] - ): ScopedEntityDefinition[Iri, SchemaState, SchemaCommand, SchemaEvent, SchemaRejection] = + ): SchemaDefinition = ScopedEntityDefinition( entityType, StateMachine(None, evaluate(validate, clock), next), diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/schemas/SchemasImpl.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/schemas/SchemasImpl.scala index 70553aed35..cd886c800c 100644 --- a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/schemas/SchemasImpl.scala +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/schemas/SchemasImpl.scala @@ -1,6 +1,6 @@ package ch.epfl.bluebrain.nexus.delta.sdk.schemas -import cats.effect.{Clock, IO} +import cats.effect.IO import cats.syntax.all._ import ch.epfl.bluebrain.nexus.delta.kernel.kamon.KamonMetricComponent import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF @@ -12,14 +12,13 @@ import ch.epfl.bluebrain.nexus.delta.sdk._ import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.implicits._ import ch.epfl.bluebrain.nexus.delta.sdk.jsonld.JsonLdSourceProcessor.JsonLdSourceResolvingParser -import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegmentRef.{Latest, Revision, Tag} import ch.epfl.bluebrain.nexus.delta.sdk.model._ import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution -import ch.epfl.bluebrain.nexus.delta.sdk.schemas.Schemas.{entityType, expandIri} +import ch.epfl.bluebrain.nexus.delta.sdk.schemas.Schemas.{entityType, expandIri, SchemaLog} import ch.epfl.bluebrain.nexus.delta.sdk.schemas.SchemasImpl.SchemasLog import ch.epfl.bluebrain.nexus.delta.sdk.schemas.model.SchemaCommand._ -import ch.epfl.bluebrain.nexus.delta.sdk.schemas.model.SchemaRejection.{RevisionNotFound, SchemaNotFound, TagNotFound} +import ch.epfl.bluebrain.nexus.delta.sdk.schemas.model.SchemaRejection.SchemaNotFound import ch.epfl.bluebrain.nexus.delta.sdk.schemas.model.{SchemaCommand, SchemaEvent, SchemaRejection, SchemaState} import ch.epfl.bluebrain.nexus.delta.sourcing._ import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject @@ -145,13 +144,7 @@ final class SchemasImpl private ( for { pc <- fetchContext.onRead(projectRef) iri <- expandIri(id.value, pc) - state <- id match { - case Latest(_) => log.stateOr(projectRef, iri, SchemaNotFound(iri, projectRef)) - case Revision(_, rev) => - log.stateOr(projectRef, iri, rev, SchemaNotFound(iri, projectRef), RevisionNotFound) - case Tag(_, tag) => - log.stateOr(projectRef, iri, tag, SchemaNotFound(iri, projectRef), TagNotFound(tag)) - } + state <- FetchSchema(log).stateOrNotFound(id, iri, projectRef) } yield state.toResource }.span("fetchSchema") @@ -173,13 +166,10 @@ object SchemasImpl { * Constructs a [[Schemas]] instance. */ final def apply( + scopedLog: SchemaLog, fetchContext: FetchContext, schemaImports: SchemaImports, - contextResolution: ResolverContextResolution, - validate: ValidateSchema, - config: SchemasConfig, - xas: Transactors, - clock: Clock[IO] + contextResolution: ResolverContextResolution )(implicit api: JsonLdApi, uuidF: UUIDF @@ -191,7 +181,7 @@ object SchemasImpl { uuidF ) new SchemasImpl( - ScopedEventLog(Schemas.definition(validate, clock), config.eventLog, xas), + scopedLog, fetchContext, schemaImports, parser diff --git a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/acls/AclsImplSpec.scala b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/acls/AclsImplSpec.scala index 928756cce6..6fac31b3f2 100644 --- a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/acls/AclsImplSpec.scala +++ b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/acls/AclsImplSpec.scala @@ -64,7 +64,7 @@ class AclsImplSpec extends CatsEffectSpec with DoobieScalaTestFixture with Cance IO.pure(minimumPermissions), Acls.findUnknownRealms(_, Set(realm, realm2)), minimumPermissions, - AclsConfig(eventLogConfig), + eventLogConfig, xas, clock ) diff --git a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/generators/ResolverResolutionGen.scala b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/generators/ResolverResolutionGen.scala index ecae57e709..423111e7d8 100644 --- a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/generators/ResolverResolutionGen.scala +++ b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/generators/ResolverResolutionGen.scala @@ -3,8 +3,9 @@ package ch.epfl.bluebrain.nexus.delta.sdk.generators import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv +import ch.epfl.bluebrain.nexus.delta.sdk.model.Fetch.Fetch import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverResolution -import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverResolution.{DeprecationCheck, Fetch} +import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverResolution.DeprecationCheck import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.model.ResolverRejection.ResolverNotFound import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Identity, ProjectRef, ResourceRef} diff --git a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/generators/ResourceResolutionGen.scala b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/generators/ResourceResolutionGen.scala index 47b53aa1f4..4c68bf21c2 100644 --- a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/generators/ResourceResolutionGen.scala +++ b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/generators/ResourceResolutionGen.scala @@ -3,7 +3,8 @@ package ch.epfl.bluebrain.nexus.delta.sdk.generators import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv -import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverResolution.{FetchResource, ResourceResolution} +import ch.epfl.bluebrain.nexus.delta.sdk.model.Fetch.FetchF +import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverResolution.ResourceResolution import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResourceResolution import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.model.ResolverRejection.ResolverNotFound import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Identity, ProjectRef, ResourceRef} @@ -19,7 +20,7 @@ object ResourceResolutionGen { */ def singleInProject[R]( projectRef: ProjectRef, - fetchResource: (ResourceRef, ProjectRef) => FetchResource[R] + fetchResource: (ResourceRef, ProjectRef) => FetchF[R] ): ResourceResolution[R] = { val resolver = ResolverGen.inProject(nxv + "in-project", projectRef) diff --git a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/organizations/OrganizationDeleterSuite.scala b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/organizations/OrganizationDeleterSuite.scala index 75b4c0eb51..c702bb4016 100644 --- a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/organizations/OrganizationDeleterSuite.scala +++ b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/organizations/OrganizationDeleterSuite.scala @@ -43,7 +43,7 @@ class OrganizationDeleterSuite extends NexusSuite with ConfigFixtures with Proje private val fields = ProjectFields(None, ApiMappings.empty, None, None) private lazy val orgs = OrganizationsImpl(ScopeInitializer.noop, eventLogConfig, xas, clock) private val permission = Permissions.resources.read - private lazy val acls = AclsImpl(IO.pure(Set(permission)), _ => IO.unit, Set(), aclsConfig, xas, clock) + private lazy val acls = AclsImpl(IO.pure(Set(permission)), _ => IO.unit, Set(), aclsConfig.eventLog, xas, clock) implicit val subject: Subject = Identity.User("Bob", Label.unsafe("realm")) implicit val uuidF: UUIDF = UUIDF.fixed(UUID.randomUUID()) diff --git a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/projects/OwnerPermissionsScopeInitializationSpec.scala b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/projects/OwnerPermissionsScopeInitializationSpec.scala index 65a761216d..a0a71021bc 100644 --- a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/projects/OwnerPermissionsScopeInitializationSpec.scala +++ b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/projects/OwnerPermissionsScopeInitializationSpec.scala @@ -3,7 +3,7 @@ package ch.epfl.bluebrain.nexus.delta.sdk.projects import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.sdk.ConfigFixtures import ch.epfl.bluebrain.nexus.delta.sdk.acls.model.{Acl, AclAddress} -import ch.epfl.bluebrain.nexus.delta.sdk.acls.{Acls, AclsConfig, AclsImpl} +import ch.epfl.bluebrain.nexus.delta.sdk.acls.{Acls, AclsImpl} import ch.epfl.bluebrain.nexus.delta.sdk.generators.{OrganizationGen, PermissionsGen, ProjectGen} import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.{Caller, ServiceAccount} import ch.epfl.bluebrain.nexus.delta.sdk.permissions.Permissions @@ -22,7 +22,7 @@ class OwnerPermissionsScopeInitializationSpec extends CatsEffectSpec with Doobie IO.pure(PermissionsGen.minimum), Acls.findUnknownRealms(_, Set(saRealm, usersRealm)), PermissionsGen.minimum, - AclsConfig(eventLogConfig), + eventLogConfig, xas, clock ) diff --git a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/resolvers/MultiResolutionSuite.scala b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/resolvers/MultiResolutionSuite.scala index 69a29cf2e2..c43e22a84c 100644 --- a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/resolvers/MultiResolutionSuite.scala +++ b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/resolvers/MultiResolutionSuite.scala @@ -6,10 +6,10 @@ import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.encoder.JsonLdEncoder import ch.epfl.bluebrain.nexus.delta.sdk.generators.{ResolverResolutionGen, ResourceGen, SchemaGen} import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.jsonld.JsonLdContent +import ch.epfl.bluebrain.nexus.delta.sdk.model.Fetch.Fetch import ch.epfl.bluebrain.nexus.delta.sdk.model.{IdSegmentRef, ResourceF} import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContextDummy import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.{ApiMappings, ProjectContext} -import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverResolution.Fetch import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.model.ResolverRejection.{InvalidResolution, InvalidResolverResolution} import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.model.ResolverResolutionRejection.ResourceNotFound import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.model.ResourceResolutionReport.ResolverReport diff --git a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/resolvers/ResolverContextResolutionSuite.scala b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/resolvers/ResolverContextResolutionSuite.scala index b4684ddfe7..72d10701ee 100644 --- a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/resolvers/ResolverContextResolutionSuite.scala +++ b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/resolvers/ResolverContextResolutionSuite.scala @@ -4,17 +4,17 @@ import akka.http.scaladsl.model.Uri import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.{contexts, nxv, schemas} -import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.{CompactedJsonLd, ExpandedJsonLd} import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.RemoteContext.StaticContext import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.RemoteContextResolutionError.RemoteContextNotAccessible import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution} +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.{CompactedJsonLd, ExpandedJsonLd} import ch.epfl.bluebrain.nexus.delta.sdk.generators.ResourceResolutionGen import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.implicits._ +import ch.epfl.bluebrain.nexus.delta.sdk.model.Fetch.FetchF import ch.epfl.bluebrain.nexus.delta.sdk.model.{ResourceF, ResourceUris, Tags} import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution.ProjectRemoteContext -import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverResolution.FetchResource import ch.epfl.bluebrain.nexus.delta.sdk.resources.model.Resource import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.User import ch.epfl.bluebrain.nexus.delta.sourcing.model.ResourceRef.Latest @@ -62,7 +62,7 @@ class ResolverContextResolutionSuite extends NexusSuite { ) ) - def fetchResource: (ResourceRef, ProjectRef) => FetchResource[Resource] = { (r: ResourceRef, p: ProjectRef) => + def fetchResource: (ResourceRef, ProjectRef) => FetchF[Resource] = { (r: ResourceRef, p: ProjectRef) => (r, p) match { case (Latest(id), `project`) if resourceId == id => IO.pure(Some(resource)) case _ => IO.none diff --git a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/resolvers/ResolverResolutionSuite.scala b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/resolvers/ResolverResolutionSuite.scala index 5c98a6a6e3..5fa1e0053e 100644 --- a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/resolvers/ResolverResolutionSuite.scala +++ b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/resolvers/ResolverResolutionSuite.scala @@ -7,8 +7,8 @@ import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.{nxv, schemas} import ch.epfl.bluebrain.nexus.delta.sdk.generators.ResolverGen import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sdk.model.Fetch.FetchF import ch.epfl.bluebrain.nexus.delta.sdk.model.{ResourceF, ResourceUris} -import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverResolution.FetchResource import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverResolutionSuite.ResourceExample import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.model.IdentityResolution.{ProvidedIdentities, UseCurrentCaller} import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.model.Resolver.CrossProjectResolver @@ -95,7 +95,7 @@ class ResolverResolutionSuite extends NexusSuite { def fetchResource( projectRef: ProjectRef - ): (ResourceRef, ProjectRef) => FetchResource[ResourceExample] = + ): (ResourceRef, ProjectRef) => FetchF[ResourceExample] = (_: ResourceRef, p: ProjectRef) => p match { case `projectRef` => IO.pure(Some(resource)) diff --git a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/ResourcesImplSpec.scala b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/ResourcesImplSpec.scala index 96c26af3a9..ed93effc4f 100644 --- a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/ResourcesImplSpec.scala +++ b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/ResourcesImplSpec.scala @@ -8,13 +8,14 @@ import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution} import ch.epfl.bluebrain.nexus.delta.sdk.generators.{ProjectGen, ResourceGen, ResourceResolutionGen, SchemaGen} import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sdk.model.Fetch.FetchF import ch.epfl.bluebrain.nexus.delta.sdk.jsonld.JsonLdRejection._ import ch.epfl.bluebrain.nexus.delta.sdk.model.{IdSegment, IdSegmentRef, Tags} import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContextDummy import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ApiMappings import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ProjectRejection.{ProjectIsDeprecated, ProjectNotFound} import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution -import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverResolution.{FetchResource, ResourceResolution} +import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverResolution.ResourceResolution import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.model.ResourceResolutionReport.ResolverReport import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.model.{ResolverResolutionRejection, ResourceResolutionReport} import ch.epfl.bluebrain.nexus.delta.sdk.resources.NexusSource.DecodingOption @@ -23,6 +24,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.resources.model.ResourceRejection._ import ch.epfl.bluebrain.nexus.delta.sdk.schemas.model.Schema import ch.epfl.bluebrain.nexus.delta.sdk.syntax._ import ch.epfl.bluebrain.nexus.delta.sdk.{ConfigFixtures, DataResource} +import ch.epfl.bluebrain.nexus.delta.sourcing.ScopedEventLog import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject import ch.epfl.bluebrain.nexus.delta.sourcing.model.ResourceRef.{Latest, Revision} import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag @@ -69,13 +71,13 @@ class ResourcesImplSpec private val schema2 = SchemaGen.schema(schema.Person, project.ref, schemaSource.removeKeys(keywords.id)) private val schema3 = SchemaGen.schema(nxv + "myschema3", project.ref, schemaSource.removeKeys(keywords.id)) - private val fetchSchema: (ResourceRef, ProjectRef) => FetchResource[Schema] = { + private val fetchSchema: (ResourceRef, ProjectRef) => FetchF[Schema] = { case (ref, _) if ref.iri == schema2.id => IO.pure(Some(SchemaGen.resourceFor(schema2, deprecated = true))) case (ref, _) if ref.iri == schema1.id => IO.pure(Some(SchemaGen.resourceFor(schema1))) case (ref, _) if ref.iri == schema3.id => IO.pure(Some(SchemaGen.resourceFor(schema3))) case _ => IO.none } - private val resourceResolution: ResourceResolution[Schema] = + private val resourceResolution: ResourceResolution[Schema] = ResourceResolutionGen.singleInProject(projectRef, fetchSchema) private val fetchContext = FetchContextDummy( @@ -93,14 +95,13 @@ class ResourcesImplSpec (r, p, _) => resources.fetch(r, p).attempt.map(_.left.map(_ => ResourceResolutionReport())) ) + private val resourceDef = Resources.definition(ValidateResource(resourceResolution), detectChanges, clock) + private lazy val resourceLog = ScopedEventLog(resourceDef, eventLogConfig, xas) + private lazy val resources: Resources = ResourcesImpl( - ValidateResource(resourceResolution), - detectChanges, + resourceLog, fetchContext, - resolverContextResolution, - config, - xas, - clock + resolverContextResolution ) private val simpleSourcePaylod = (id: IdSegment) => json"""{ "@id": "$id", "some": "content" }""" diff --git a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/ValidateResourceSuite.scala b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/ValidateResourceSuite.scala index 841b524570..6e33db8e10 100644 --- a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/ValidateResourceSuite.scala +++ b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/ValidateResourceSuite.scala @@ -9,7 +9,8 @@ import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.{CompactedJsonLd, ExpandedJsonLd import ch.epfl.bluebrain.nexus.delta.sdk.generators.{ResourceResolutionGen, SchemaGen} import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.jsonld.JsonLdAssembly -import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverResolution.{FetchResource, ResourceResolution} +import ch.epfl.bluebrain.nexus.delta.sdk.model.Fetch.FetchF +import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverResolution.ResourceResolution import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.model.ResourceResolutionReport.ResolverReport import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.model.{ResolverResolutionRejection, ResourceResolutionReport} import ch.epfl.bluebrain.nexus.delta.sdk.resources.model.ResourceRejection._ @@ -56,12 +57,12 @@ class ValidateResourceSuite extends NexusSuite { private val unconstrained = ResourceRef.Revision(schemas.resources, 1) - private val fetchSchema: (ResourceRef, ProjectRef) => FetchResource[Schema] = { + private val fetchSchema: (ResourceRef, ProjectRef) => FetchF[Schema] = { case (ref, p) if ref.iri == schemaId && p == project => schema.map(Some(_)) case (ref, p) if ref.iri == deprecatedSchemaId && p == project => deprecatedSchema.map(Some(_)) case _ => IO.none } - private val schemaResolution: ResourceResolution[Schema] = + private val schemaResolution: ResourceResolution[Schema] = ResourceResolutionGen.singleInProject(project, fetchSchema) private def sourceWithId(id: Iri) = diff --git a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/schemas/SchemasImplSuite.scala b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/schemas/SchemasImplSuite.scala index 0e6403bd06..bd06c3827a 100644 --- a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/schemas/SchemasImplSuite.scala +++ b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/schemas/SchemasImplSuite.scala @@ -10,8 +10,8 @@ import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteCon import ch.epfl.bluebrain.nexus.delta.rdf.shacl.ShaclShapesGraph import ch.epfl.bluebrain.nexus.delta.sdk.ConfigFixtures import ch.epfl.bluebrain.nexus.delta.sdk.generators.{ProjectGen, SchemaGen} -import ch.epfl.bluebrain.nexus.delta.sdk.jsonld.JsonLdRejection.UnexpectedId import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sdk.jsonld.JsonLdRejection.UnexpectedId import ch.epfl.bluebrain.nexus.delta.sdk.model.{IdSegmentRef, Tags} import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContextDummy import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ApiMappings @@ -20,6 +20,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution import ch.epfl.bluebrain.nexus.delta.sdk.schemas.model.Schema import ch.epfl.bluebrain.nexus.delta.sdk.schemas.model.SchemaRejection._ import ch.epfl.bluebrain.nexus.delta.sdk.syntax._ +import ch.epfl.bluebrain.nexus.delta.sourcing.ScopedEventLog import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Identity, Label, ProjectRef} @@ -75,8 +76,11 @@ class SchemasImplSuite extends NexusSuite with Doobie.Fixture with ConfigFixture private val fetchContext = FetchContextDummy(Map(project.ref -> project.context), Set(projectDeprecated.ref)) private val config = SchemasConfig(eventLogConfig) + private val schemaDef = Schemas.definition(ValidateSchema.apply, clock) + private lazy val schemaLog = ScopedEventLog(schemaDef, config.eventLog, xas) + private lazy val schemas: Schemas = - SchemasImpl(fetchContext, schemaImports, resolverContextResolution, ValidateSchema.apply, config, xas, clock) + SchemasImpl(schemaLog, fetchContext, schemaImports, resolverContextResolution) private def schemaSourceWithId(id: Iri) = { source deepMerge json"""{"@id": "$id"}""" diff --git a/delta/sourcing-psql/src/main/scala/ch/epfl/bluebrain/nexus/delta/sourcing/ScopedEventLog.scala b/delta/sourcing-psql/src/main/scala/ch/epfl/bluebrain/nexus/delta/sourcing/ScopedEventLog.scala index 096f98cb95..eda2440165 100644 --- a/delta/sourcing-psql/src/main/scala/ch/epfl/bluebrain/nexus/delta/sourcing/ScopedEventLog.scala +++ b/delta/sourcing-psql/src/main/scala/ch/epfl/bluebrain/nexus/delta/sourcing/ScopedEventLog.scala @@ -33,55 +33,8 @@ import scala.concurrent.duration.FiniteDuration * Unsuccessful commands result in rejections returned to the caller context without any events being generated or * state transitions applied. */ -trait ScopedEventLog[Id, S <: ScopedState, Command, E <: ScopedEvent, Rejection <: Throwable] { - - /** - * Get the latest state for the entity with the given __id__ in the given project - * - * @param ref - * the project the entity belongs in - * @param id - * the entity identifier - * @param notFound - * if no state is found, fails with this rejection - */ - def stateOr[R <: Rejection](ref: ProjectRef, id: Id, notFound: => R): IO[S] - - /** - * Get the state for the entity with the given __id__ at the given __tag__ in the given project - * @param ref - * the project the entity belongs in - * @param id - * the entity identifier - * @param tag - * the tag - * @param notFound - * if no state is found, fails with this rejection - * @param tagNotFound - * if no state is found with the provided tag, fails with this rejection - */ - def stateOr[R <: Rejection](ref: ProjectRef, id: Id, tag: Tag, notFound: => R, tagNotFound: => R): IO[S] - - /** - * Get the state for the entity with the given __id__ at the given __revision__ in the given project - * @param ref - * the project the entity belongs in - * @param id - * the entity identifier - * @param rev - * the revision - * @param notFound - * if no state is found, fails with this rejection - * @param invalidRevision - * if the revision of the resulting state does not match with the one provided - */ - def stateOr[R <: Rejection]( - ref: ProjectRef, - id: Id, - rev: Int, - notFound: => R, - invalidRevision: (Int, Int) => R - ): IO[S] +trait ScopedEventLog[Id, S <: ScopedState, Command, E <: ScopedEvent, Rejection <: Throwable] + extends ScopedEventLogReadOnly[Id, S, Rejection] { /** * Evaluates the argument __command__ in the context of entity identified by __id__. @@ -113,50 +66,6 @@ trait ScopedEventLog[Id, S <: ScopedState, Command, E <: ScopedEvent, Rejection */ def dryRun(ref: ProjectRef, id: Id, command: Command): IO[(E, S)] - /** - * Allow to stream all latest states within [[Elem.SuccessElem]] s without applying transformation - * @param scope - * to filter returned states - * @param offset - * offset to start from - */ - def currentStates(scope: Scope, offset: Offset): SuccessElemStream[S] - - /** - * Allow to stream all latest states from the beginning within [[Elem.SuccessElem]] s without applying transformation - * @param scope - * to filter returned states - */ - def currentStates(scope: Scope): SuccessElemStream[S] = currentStates(scope, Offset.Start) - - /** - * Allow to stream all current states from the provided offset - * @param scope - * to filter returned states - * @param offset - * offset to start from - * @param f - * the function to apply on each state - */ - def currentStates[T](scope: Scope, offset: Offset, f: S => T): Stream[IO, T] - - /** - * Allow to stream all current states from the beginning - * @param scope - * to filter returned states - * @param f - * the function to apply on each state - */ - def currentStates[T](scope: Scope, f: S => T): Stream[IO, T] = currentStates(scope, Offset.Start, f) - - /** - * Stream the state changes continuously from the provided offset. - * @param scope - * to filter returned states - * @param offset - * the start offset - */ - def states(scope: Scope, offset: Offset): SuccessElemStream[S] } object ScopedEventLog { diff --git a/delta/sourcing-psql/src/main/scala/ch/epfl/bluebrain/nexus/delta/sourcing/ScopedEventLogReadOnly.scala b/delta/sourcing-psql/src/main/scala/ch/epfl/bluebrain/nexus/delta/sourcing/ScopedEventLogReadOnly.scala new file mode 100644 index 0000000000..9c333428a3 --- /dev/null +++ b/delta/sourcing-psql/src/main/scala/ch/epfl/bluebrain/nexus/delta/sourcing/ScopedEventLogReadOnly.scala @@ -0,0 +1,106 @@ +package ch.epfl.bluebrain.nexus.delta.sourcing + +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.sourcing.model._ +import ch.epfl.bluebrain.nexus.delta.sourcing.offset.Offset +import ch.epfl.bluebrain.nexus.delta.sourcing.state.State.ScopedState +import fs2.Stream + +/** + * Read-only event log for project-scoped entities + */ +trait ScopedEventLogReadOnly[Id, S <: ScopedState, Rejection <: Throwable] { + + /** + * Get the latest state for the entity with the given __id__ in the given project + * + * @param ref + * the project the entity belongs in + * @param id + * the entity identifier + * @param notFound + * if no state is found, fails with this rejection + */ + def stateOr[R <: Rejection](ref: ProjectRef, id: Id, notFound: => R): IO[S] + + /** + * Get the state for the entity with the given __id__ at the given __tag__ in the given project + * @param ref + * the project the entity belongs in + * @param id + * the entity identifier + * @param tag + * the tag + * @param notFound + * if no state is found, fails with this rejection + * @param tagNotFound + * if no state is found with the provided tag, fails with this rejection + */ + def stateOr[R <: Rejection](ref: ProjectRef, id: Id, tag: Tag, notFound: => R, tagNotFound: => R): IO[S] + + /** + * Get the state for the entity with the given __id__ at the given __revision__ in the given project + * @param ref + * the project the entity belongs in + * @param id + * the entity identifier + * @param rev + * the revision + * @param notFound + * if no state is found, fails with this rejection + * @param invalidRevision + * if the revision of the resulting state does not match with the one provided + */ + def stateOr[R <: Rejection]( + ref: ProjectRef, + id: Id, + rev: Int, + notFound: => R, + invalidRevision: (Int, Int) => R + ): IO[S] + + /** + * Allow to stream all latest states within [[Elem.SuccessElem]] s without applying transformation + * @param scope + * to filter returned states + * @param offset + * offset to start from + */ + def currentStates(scope: Scope, offset: Offset): SuccessElemStream[S] + + /** + * Allow to stream all latest states from the beginning within [[Elem.SuccessElem]] s without applying transformation + * @param scope + * to filter returned states + */ + def currentStates(scope: Scope): SuccessElemStream[S] = currentStates(scope, Offset.Start) + + /** + * Allow to stream all current states from the provided offset + * @param scope + * to filter returned states + * @param offset + * offset to start from + * @param f + * the function to apply on each state + */ + def currentStates[T](scope: Scope, offset: Offset, f: S => T): Stream[IO, T] + + /** + * Allow to stream all current states from the beginning + * @param scope + * to filter returned states + * @param f + * the function to apply on each state + */ + def currentStates[T](scope: Scope, f: S => T): Stream[IO, T] = currentStates(scope, Offset.Start, f) + + /** + * Stream the state changes continuously from the provided offset. + * @param scope + * to filter returned states + * @param offset + * the start offset + */ + def states(scope: Scope, offset: Offset): SuccessElemStream[S] +} diff --git a/delta/sourcing-psql/src/main/scala/ch/epfl/bluebrain/nexus/delta/sourcing/Transactors.scala b/delta/sourcing-psql/src/main/scala/ch/epfl/bluebrain/nexus/delta/sourcing/Transactors.scala index dfdce0e0f6..4efa8a459f 100644 --- a/delta/sourcing-psql/src/main/scala/ch/epfl/bluebrain/nexus/delta/sourcing/Transactors.scala +++ b/delta/sourcing-psql/src/main/scala/ch/epfl/bluebrain/nexus/delta/sourcing/Transactors.scala @@ -105,19 +105,19 @@ object Transactors { ): Resource[IO, Transactors] = { def transactor(access: DatabaseAccess, readOnly: Boolean, poolName: String): Resource[IO, HikariTransactor[IO]] = { for { - ec <- ExecutionContexts.fixedThreadPool[IO](access.poolSize) - dataSource = { - val ds = new HikariDataSource - ds.setJdbcUrl(s"jdbc:postgresql://${access.host}:${access.port}/") - ds.setUsername(config.username) - ds.setPassword(config.password.value) - ds.setDriverClassName("org.postgresql.Driver") - ds.setMaximumPoolSize(access.poolSize) - ds.setPoolName(poolName) - ds.setAutoCommit(false) - ds.setReadOnly(readOnly) - ds - } + ec <- ExecutionContexts.fixedThreadPool[IO](access.poolSize) + dataSource <- Resource.make[IO, HikariDataSource](IO.delay { + val ds = new HikariDataSource + ds.setJdbcUrl(s"jdbc:postgresql://${access.host}:${access.port}/") + ds.setUsername(config.username) + ds.setPassword(config.password.value) + ds.setDriverClassName("org.postgresql.Driver") + ds.setMaximumPoolSize(access.poolSize) + ds.setPoolName(poolName) + ds.setAutoCommit(false) + ds.setReadOnly(readOnly) + ds + })(ds => IO.delay(ds.close())) } yield HikariTransactor[IO](dataSource, ec, None) } diff --git a/delta/sourcing-psql/src/main/scala/ch/epfl/bluebrain/nexus/delta/sourcing/state/ScopedStateGet.scala b/delta/sourcing-psql/src/main/scala/ch/epfl/bluebrain/nexus/delta/sourcing/state/ScopedStateGet.scala index b377cbf9ce..1c6d18e6a2 100644 --- a/delta/sourcing-psql/src/main/scala/ch/epfl/bluebrain/nexus/delta/sourcing/state/ScopedStateGet.scala +++ b/delta/sourcing-psql/src/main/scala/ch/epfl/bluebrain/nexus/delta/sourcing/state/ScopedStateGet.scala @@ -8,11 +8,22 @@ import doobie.{ConnectionIO, Get, Put} object ScopedStateGet { def apply[Id: Put, S: Get](tpe: EntityType, project: ProjectRef, id: Id, tag: Tag): ConnectionIO[Option[S]] = - sql"""SELECT value FROM scoped_states WHERE type = $tpe AND org = ${project.organization} AND project = ${project.project} AND id = $id AND tag = $tag""" + sql"""SELECT value FROM scoped_states WHERE type = $tpe AND org = ${project.organization} AND project = ${project.project} AND id = $id AND tag = $tag""" + .query[S] + .option + + def apply[Id: Put, S: Get](tpe: EntityType, project: ProjectRef, id: Id, rev: Int): ConnectionIO[Option[S]] = + sql"""SELECT value FROM scoped_states WHERE type = $tpe AND org = ${project.organization} AND project = ${project.project} AND id = $id AND rev = $rev""" .query[S] .option def latest[Id: Put, S: Get](tpe: EntityType, project: ProjectRef, id: Id): ConnectionIO[Option[S]] = apply(tpe, project, id, Latest) + def tag[Id: Put, S: Get](tpe: EntityType, project: ProjectRef, id: Id, tag: Tag): ConnectionIO[Option[S]] = + apply(tpe, project, id, tag) + + def rev[Id: Put, S: Get](tpe: EntityType, project: ProjectRef, id: Id, rev: Int): ConnectionIO[Option[S]] = + apply(tpe, project, id, rev) + } diff --git a/delta/testkit/src/main/scala/ch/epfl/bluebrain/nexus/testkit/scalatest/ce/CatsIOValues.scala b/delta/testkit/src/main/scala/ch/epfl/bluebrain/nexus/testkit/scalatest/ce/CatsIOValues.scala index a9f35012bc..d9e2e2ab35 100644 --- a/delta/testkit/src/main/scala/ch/epfl/bluebrain/nexus/testkit/scalatest/ce/CatsIOValues.scala +++ b/delta/testkit/src/main/scala/ch/epfl/bluebrain/nexus/testkit/scalatest/ce/CatsIOValues.scala @@ -14,10 +14,7 @@ trait CatsIOValues { implicit final class CatsIOValuesOps[A](private val io: IO[A]) { def accepted(implicit pos: source.Position): A = { - io.attempt.unsafeRunTimed(45.seconds).getOrElse(fail("IO timed out during .accepted call")) match { - case Left(e) => fail(s"IO failed when it was expected to succeed $e.", e) - case Right(value) => value - } + io.unsafeRunTimed(45.seconds).getOrElse(fail("IO timed out during .accepted call")) } def rejected(implicit pos: source.Position): Throwable = rejectedWith[Throwable] diff --git a/infra/aws.conf b/infra/aws.conf new file mode 100644 index 0000000000..04013db05e --- /dev/null +++ b/infra/aws.conf @@ -0,0 +1,176 @@ +app { + description { + env = "production" + } + + database { + read = ${app.defaults.database.access} { + pool-size = 15 + } + write = ${app.defaults.database.access} { + pool-size = 15 + } + streaming = ${app.defaults.database.access} { + pool-size = 50 + } + + name = "nexus_user" + username = "nexus_user" + password = ${POSTGRES_PASSWORD} + + tables-autocreate = true + } + + projections { + batch { + max-elements = 500 + } + } + + defaults { + database { + access { + host = ${POSTGRES_HOST} + port = 5432 + } + } + + query { + batch-size = 30 + } + + pagination { + size-limit = 2000 + } + + indexing { + prefix = "nexus" + } + } + + fusion { + enable-redirects = true + base = "https://sbo-nexus-fusion.shapes-registry.org/nexus/web" + } + + http { + interface = 0.0.0.0 + base-uri = "https://sbo-nexus-delta.shapes-registry.org/v1" + } + + monitoring { + trace { + sampler = "never" + } + } + + projects { + deletion { + enabled = true + } + } +} + +plugins { + blazegraph { + enabled = true + base = ${BLAZEGRAPH_ENDPOINT} + batch { + max-elements = 2 + } + } + + elasticsearch { + enabled = true + base = ${ELASTICSEARCH_ENDPOINT} + credentials { + username = "elastic" + password = ${ELASTICSEARCH_PASSWORD} + } + batch { + max-elements = 10 + } + } + + composite-views { + enabled = true + elasticsearch-batch { + max-elements = 10 + } + blazegraph-batch { + max-elements = 2 + } + sink-config = "batch" + } + + graph-analytics { + enabled = true + batch = { + max-elements = 10 + } + } + + jira { + enabled = false + base = "https://bbpteam.epfl.ch/project/issues" + } + + project-deletion { + enabled = false + } + + search { + enabled = true + fields = /opt/search-config/fields.json + indexing { + context = /opt/search-config/search-context.json + mapping = /opt/search-config/mapping.json + settings = /opt/search-config/settings.json + query = /opt/search-config/construct-query.sparql + resource-types = /opt/search-config/resource-types.json + } + suites = { + sbo = [ + "public/forge", + "public/hippocampus", + "public/hippocampus-hub", + "public/multi-vesicular-release", + "public/ngv", + "public/ngv-anatomy", + "public/nvg", + "public/sscx", + "public/thalamus", + "public/topological-sampling", + "bbp/lnmce", + "bbp/ncmv3", + "bbp/hippocampus", + "bbp-external/seu", + "bbp/mouselight" + ] + } + } + + storage { + enabled = true + storages { + disk { + default-volume = /opt/disk-storage + } + remote-disk { + enabled = true + default-endpoint = "http://sbo-poc-pcluster.shapes-registry.org:8081/v1" + credentials = { + type = client-credentials + user = nexus-delta + realm = SBO + password = ${REMOTE_STORAGE_PASSWORD} + } + } + } + } + + serivce-account { + realm = "SBO" + subject = "nexus-delta" + } +} \ No newline at end of file diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/ContextWiring.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/ContextWiring.scala new file mode 100644 index 0000000000..5aaea9751f --- /dev/null +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/ContextWiring.scala @@ -0,0 +1,58 @@ +package ch.epfl.bluebrain.nexus.ship + +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.kernel.utils.ClasspathResourceLoader +import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.contexts +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution} +import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck +import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext +import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution +import ch.epfl.bluebrain.nexus.delta.sdk.resources.FetchResource +import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors +import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig +import ch.epfl.bluebrain.nexus.ship.acls.AclWiring +import ch.epfl.bluebrain.nexus.ship.resolvers.ResolverWiring + +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.{contexts => esContexts} +import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.{contexts => bgContexts} + +object ContextWiring { + + implicit private val loader: ClasspathResourceLoader = ClasspathResourceLoader.withContext(getClass) + + def remoteContextResolution: IO[RemoteContextResolution] = + for { + pipelineCtx <- ContextValue.fromFile("contexts/pipeline.json") + shaclCtx <- ContextValue.fromFile("contexts/shacl.json") + schemasMetaCtx <- ContextValue.fromFile("contexts/schemas-metadata.json") + elasticsearchCtx <- ContextValue.fromFile("contexts/elasticsearch.json") + blazegraphCtx <- ContextValue.fromFile("contexts/sparql.json") + } yield RemoteContextResolution.fixed( + // Delta + contexts.pipeline -> pipelineCtx, + // Schema + contexts.shacl -> shaclCtx, + contexts.schemasMetadata -> schemasMetaCtx, + // ElasticSearch + esContexts.elasticsearch -> elasticsearchCtx, + // Blazegraph + bgContexts.blazegraph -> blazegraphCtx + ) + + def resolverContextResolution( + fetchResource: FetchResource, + fetchContext: FetchContext, + config: EventLogConfig, + clock: EventClock, + xas: Transactors + )(implicit jsonLdApi: JsonLdApi): IO[ResolverContextResolution] = { + val aclCheck = AclCheck(AclWiring.acls(config, clock, xas)) + val resolvers = ResolverWiring.resolvers(fetchContext, config, clock, xas) + + for { + rcr <- remoteContextResolution + } yield ResolverContextResolution(aclCheck, resolvers, rcr, fetchResource) + } + +} diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/EventClock.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/EventClock.scala index 27d74b5326..c7f5c56d39 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/EventClock.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/EventClock.scala @@ -21,7 +21,9 @@ class EventClock(instant: Ref[IO, Instant]) extends Clock[IO] { override def realTime: IO[FiniteDuration] = toDuration private def toDuration: IO[FiniteDuration] = instant.get.map { i => - FiniteDuration(i.toEpochMilli, TimeUnit.MILLISECONDS) + val seconds = FiniteDuration(i.getEpochSecond, TimeUnit.SECONDS) + val nanos = FiniteDuration(i.getNano, TimeUnit.NANOSECONDS) + seconds + nanos } } diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/Main.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/Main.scala index c353f4d64f..66e1cbdc9d 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/Main.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/Main.scala @@ -1,27 +1,14 @@ package ch.epfl.bluebrain.nexus.ship -import cats.effect.{Clock, ExitCode, IO} +import cats.effect.{ExitCode, IO} import cats.syntax.all._ import ch.epfl.bluebrain.nexus.delta.kernel.Logger -import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF -import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.{JsonLdApi, JsonLdJavaApi} -import ch.epfl.bluebrain.nexus.delta.sdk.organizations.FetchActiveOrganization -import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext -import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ApiMappings -import ch.epfl.bluebrain.nexus.delta.sdk.quotas.Quotas import ch.epfl.bluebrain.nexus.delta.ship.BuildInfo -import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors import ch.epfl.bluebrain.nexus.delta.sourcing.offset.Offset import ch.epfl.bluebrain.nexus.ship.config.ShipConfig -import ch.epfl.bluebrain.nexus.ship.model.InputEvent -import ch.epfl.bluebrain.nexus.ship.organizations.OrganizationProvider -import ch.epfl.bluebrain.nexus.ship.projects.ProjectProcessor -import ch.epfl.bluebrain.nexus.ship.resolvers.ResolverProcessor import com.monovore.decline.Opts import com.monovore.decline.effect.CommandIOApp -import fs2.Stream -import fs2.io.file.{Files, Path} -import io.circe.parser._ +import fs2.io.file.Path object Main extends CommandIOApp( @@ -54,38 +41,11 @@ object Main override def main: Opts[IO[ExitCode]] = (run orElse showConfig) .map { - case Run(file, config, _) => run(file, config) + case Run(file, config, _) => new RunShip().run(file, config) case ShowConfig(config) => showConfig(config) } .map(_.as(ExitCode.Success)) - private[ship] def run(file: Path, config: Option[Path]): IO[ImportReport] = { - val clock = Clock[IO] - val uuidF = UUIDF.random - // Resources may have been created with different configurations so we adopt the lenient one for the import - implicit val jsonLdApi: JsonLdApi = JsonLdJavaApi.lenient - for { - _ <- logger.info(s"Running the import with file $file, config $config and from offset $offset") - config <- ShipConfig.load(config) - report <- Transactors.init(config.database).use { xas => - val orgProvider = - OrganizationProvider(config.eventLog, config.serviceAccount.value, xas, clock)(uuidF) - val fetchContext = FetchContext(ApiMappings.empty, xas, Quotas.disabled) - val eventLogConfig = config.eventLog - val baseUri = config.baseUri - for { - // Provision organizations - _ <- orgProvider.create(config.organizations.values) - events = eventStream(file) - fetchActiveOrg = FetchActiveOrganization(xas) - projectProcessor <- ProjectProcessor(fetchActiveOrg, eventLogConfig, xas)(baseUri) - resolverProcessor <- ResolverProcessor(fetchContext, eventLogConfig, xas) - report <- EventProcessor.run(events, projectProcessor, resolverProcessor) - } yield report - } - } yield report - } - private[ship] def showConfig(config: Option[Path]) = for { _ <- logger.info(s"Showing reconciled config") @@ -93,13 +53,6 @@ object Main _ <- logger.info(config.root().render()) } yield () - private def eventStream(file: Path): Stream[IO, InputEvent] = - Files[IO].readUtf8Lines(file).zipWithIndex.evalMap { case (line, index) => - IO.fromEither(decode[InputEvent](line)).onError { err => - logger.error(err)(s"Error parsing to event at line $index") - } - } - sealed private trait Command final private case class Run(file: Path, config: Option[Path], offset: Offset) extends Command diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala new file mode 100644 index 0000000000..f5acee3a10 --- /dev/null +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala @@ -0,0 +1,91 @@ +package ch.epfl.bluebrain.nexus.ship + +import cats.effect.{Clock, IO} +import ch.epfl.bluebrain.nexus.delta.kernel.Logger +import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.{JsonLdApi, JsonLdJavaApi} +import ch.epfl.bluebrain.nexus.delta.sdk.organizations.FetchActiveOrganization +import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext +import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ApiMappings +import ch.epfl.bluebrain.nexus.delta.sdk.quotas.Quotas +import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors +import ch.epfl.bluebrain.nexus.ship.config.ShipConfig +import ch.epfl.bluebrain.nexus.ship.model.InputEvent +import ch.epfl.bluebrain.nexus.ship.organizations.OrganizationProvider +import ch.epfl.bluebrain.nexus.ship.projects.ProjectProcessor +import ch.epfl.bluebrain.nexus.ship.resolvers.ResolverProcessor +import ch.epfl.bluebrain.nexus.ship.resources.{ResourceProcessor, ResourceWiring} +import ch.epfl.bluebrain.nexus.ship.schemas.{SchemaProcessor, SchemaWiring} +import ch.epfl.bluebrain.nexus.ship.views.{BlazegraphViewProcessor, ElasticSearchViewProcessor} +import fs2.Stream +import fs2.io.file.{Files, Path} +import io.circe.parser.decode + +class RunShip { + + private val logger = Logger[RunShip] + + def run(file: Path, config: Option[Path]): IO[ImportReport] = { + val clock = Clock[IO] + val uuidF = UUIDF.random + // Resources may have been created with different configurations so we adopt the lenient one for the import + implicit val jsonLdApi: JsonLdApi = JsonLdJavaApi.lenient + for { + _ <- logger.info(s"Running the import with file $file, config $config") + config <- ShipConfig.load(config) + report <- Transactors.init(config.database).use { xas => + val orgProvider = + OrganizationProvider(config.eventLog, config.serviceAccount.value, xas, clock)(uuidF) + val fetchContext = FetchContext(ApiMappings.empty, xas, Quotas.disabled) + val eventLogConfig = config.eventLog + val baseUri = config.baseUri + for { + // Provision organizations + _ <- orgProvider.create(config.organizations.values) + events = eventStream(file) + fetchActiveOrg = FetchActiveOrganization(xas) + // Wiring + eventClock <- EventClock.init() + (schemaLog, fetchSchema) <- SchemaWiring(config.eventLog, eventClock, xas, jsonLdApi) + (resourceLog, fetchResource) = + ResourceWiring(fetchContext, fetchSchema, eventLogConfig, eventClock, xas) + rcr <- ContextWiring + .resolverContextResolution(fetchResource, fetchContext, eventLogConfig, eventClock, xas) + schemaImports = SchemaWiring.schemaImports( + fetchResource, + fetchSchema, + fetchContext, + eventLogConfig, + eventClock, + xas + ) + // Processors + projectProcessor <- ProjectProcessor(fetchActiveOrg, eventLogConfig, eventClock, xas)(baseUri) + resolverProcessor = ResolverProcessor(fetchContext, eventLogConfig, eventClock, xas) + schemaProcessor = SchemaProcessor(schemaLog, fetchContext, schemaImports, rcr, eventClock) + resourceProcessor = ResourceProcessor(resourceLog, fetchContext, eventClock) + esViewsProcessor <- ElasticSearchViewProcessor(fetchContext, rcr, eventLogConfig, eventClock, xas) + bgViewsProcessor = BlazegraphViewProcessor(fetchContext, rcr, eventLogConfig, eventClock, xas) + report <- EventProcessor + .run( + events, + projectProcessor, + resolverProcessor, + schemaProcessor, + resourceProcessor, + esViewsProcessor, + bgViewsProcessor + ) + } yield report + } + } yield report + } + + private def eventStream(file: Path): Stream[IO, InputEvent] = + Files[IO].readUtf8Lines(file).zipWithIndex.evalMap { case (line, index) => + IO.fromEither(decode[InputEvent](line)).onError { err => + logger.error(err)(s"Error parsing to event at line $index") + } + } + +} diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/acls/AclWiring.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/acls/AclWiring.scala new file mode 100644 index 0000000000..967962ff5d --- /dev/null +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/acls/AclWiring.scala @@ -0,0 +1,23 @@ +package ch.epfl.bluebrain.nexus.ship.acls + +import cats.effect.{Clock, IO} +import ch.epfl.bluebrain.nexus.delta.sdk.acls.{Acls, AclsImpl} +import ch.epfl.bluebrain.nexus.delta.sdk.permissions.model.Permission +import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors +import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig + +object AclWiring { + + def acls(config: EventLogConfig, clock: Clock[IO], xas: Transactors): Acls = { + val permissionSet = Set(Permission.unsafe("resources/read")) + AclsImpl( + IO.pure(permissionSet), + AclsImpl.findUnknownRealms(xas), + permissionSet, + config, + xas, + clock + ) + } + +} diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala index dcc8d85a43..f149590cfa 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala @@ -63,11 +63,10 @@ final class ProjectProcessor private (projects: Projects, clock: EventClock, uui object ProjectProcessor { private val logger = Logger[ProjectProcessor] - def apply(fetchActiveOrg: FetchActiveOrganization, config: EventLogConfig, xas: Transactors)(implicit - base: BaseUri + def apply(fetchActiveOrg: FetchActiveOrganization, config: EventLogConfig, clock: EventClock, xas: Transactors)( + implicit base: BaseUri ): IO[ProjectProcessor] = for { - clock <- EventClock.init() uuidF <- EventUUIDF.init() } yield { val disableDeletion: ValidateProjectDeletion = (p: ProjectRef) => IO.raiseError(ProjectDeletionIsNotAllowed(p)) diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resolvers/ResolverProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resolvers/ResolverProcessor.scala index 00282709ff..41873888d5 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resolvers/ResolverProcessor.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resolvers/ResolverProcessor.scala @@ -2,11 +2,10 @@ package ch.epfl.bluebrain.nexus.ship.resolvers import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.Logger -import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext -import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.{ResolverContextResolution, Resolvers, ResolversImpl} +import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.Resolvers import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.model.IdentityResolution._ import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.model.ResolverEvent._ import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.model.ResolverRejection.{IncorrectRev, ResourceAlreadyExists} @@ -17,7 +16,7 @@ import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject import ch.epfl.bluebrain.nexus.delta.sourcing.model.{EntityType, Identity} import ch.epfl.bluebrain.nexus.ship.resolvers.ResolverProcessor.logger -import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, FailingUUID, ImportStatus} +import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, ImportStatus} import io.circe.Decoder class ResolverProcessor private (resolvers: Resolvers, clock: EventClock) extends EventProcessor[ResolverEvent] { @@ -53,7 +52,7 @@ class ResolverProcessor private (resolvers: Resolvers, clock: EventClock) extend }.redeemWith( { case a: ResourceAlreadyExists => logger.warn(a)("The resolver already exists").as(ImportStatus.Dropped) - case i: IncorrectRev => logger.warn(i)("An incorrect revision as been provided").as(ImportStatus.Dropped) + case i: IncorrectRev => logger.warn(i)("An incorrect revision has been provided").as(ImportStatus.Dropped) case other => IO.raiseError(other) }, _ => IO.pure(ImportStatus.Success) @@ -77,18 +76,10 @@ object ResolverProcessor { def apply( fetchContext: FetchContext, config: EventLogConfig, + clock: EventClock, xas: Transactors - )(implicit api: JsonLdApi): IO[ResolverProcessor] = - EventClock.init().map { clock => - implicit val uuidF: UUIDF = FailingUUID - val resolvers = ResolversImpl( - fetchContext, - // We rely on the parsed values and not on the original value - ResolverContextResolution.never, - config, - xas, - clock - ) - new ResolverProcessor(resolvers, clock) - } + )(implicit api: JsonLdApi): ResolverProcessor = { + val resolvers = ResolverWiring.resolvers(fetchContext, config, clock, xas) + new ResolverProcessor(resolvers, clock) + } } diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resolvers/ResolverWiring.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resolvers/ResolverWiring.scala new file mode 100644 index 0000000000..f04f9813eb --- /dev/null +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resolvers/ResolverWiring.scala @@ -0,0 +1,29 @@ +package ch.epfl.bluebrain.nexus.ship.resolvers + +import cats.effect.IO +import cats.effect.kernel.Clock +import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi +import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext +import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.{ResolverContextResolution, Resolvers, ResolversImpl} +import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors +import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig +import ch.epfl.bluebrain.nexus.ship.FailingUUID + +object ResolverWiring { + + def resolvers(fetchContext: FetchContext, config: EventLogConfig, clock: Clock[IO], xas: Transactors)(implicit + jsonLdApi: JsonLdApi + ): Resolvers = { + implicit val uuidF: UUIDF = FailingUUID + ResolversImpl( + fetchContext, + // We rely on the parsed values and not on the original value + ResolverContextResolution.never, + config, + xas, + clock + ) + } + +} diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resources/ResourceProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resources/ResourceProcessor.scala new file mode 100644 index 0000000000..9021a4cabf --- /dev/null +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resources/ResourceProcessor.scala @@ -0,0 +1,86 @@ +package ch.epfl.bluebrain.nexus.ship.resources + +import cats.effect.IO +import cats.implicits.catsSyntaxOptionId +import ch.epfl.bluebrain.nexus.delta.kernel.Logger +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sdk.model.{IdSegment, IdSegmentRef} +import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext +import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution +import ch.epfl.bluebrain.nexus.delta.sdk.resources.Resources.ResourceLog +import ch.epfl.bluebrain.nexus.delta.sdk.resources._ +import ch.epfl.bluebrain.nexus.delta.sdk.resources.model.ResourceEvent +import ch.epfl.bluebrain.nexus.delta.sdk.resources.model.ResourceEvent._ +import ch.epfl.bluebrain.nexus.delta.sdk.resources.model.ResourceRejection.{IncorrectRev, ResourceAlreadyExists} +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{EntityType, ResourceRef} +import ch.epfl.bluebrain.nexus.ship.resources.ResourceProcessor.logger +import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, ImportStatus} +import io.circe.Decoder + +class ResourceProcessor private (resources: Resources, clock: EventClock) extends EventProcessor[ResourceEvent] { + + override def resourceType: EntityType = Resources.entityType + + override def decoder: Decoder[ResourceEvent] = ResourceEvent.serializer.codec + + override def evaluate(event: ResourceEvent): IO[ImportStatus] = + for { + _ <- clock.setInstant(event.instant) + result <- evaluateInternal(event) + } yield result + + private def evaluateInternal(event: ResourceEvent): IO[ImportStatus] = { + implicit val s: Subject = event.subject + implicit val c: Caller = Caller(s, Set.empty) + val cRev = event.rev - 1 + + implicit class ResourceRefOps(ref: ResourceRef) { + def toIdSegment: IdSegment = IdSegmentRef(ref).value + } + + event match { + case e: ResourceCreated => + resources.create(e.id, e.project, e.schema.toIdSegment, e.source, e.tag) + case e: ResourceUpdated => + resources.update(e.id, event.project, e.schema.toIdSegment.some, cRev, e.source, e.tag) + case e: ResourceSchemaUpdated => + resources.updateAttachedSchema(e.id, e.project, e.schema.toIdSegment) + case e: ResourceRefreshed => + resources.refresh(e.id, e.project, e.schema.toIdSegment.some) + case e: ResourceTagAdded => + resources.tag(e.id, e.project, None, e.tag, e.targetRev, cRev) + case e: ResourceTagDeleted => + resources.deleteTag(e.id, e.project, None, e.tag, cRev) + case e: ResourceDeprecated => + resources.deprecate(e.id, e.project, None, cRev) + case e: ResourceUndeprecated => + resources.undeprecate(e.id, e.project, None, cRev) + } + }.redeemWith( + { + case a: ResourceAlreadyExists => logger.warn(a)("The resource already exists").as(ImportStatus.Dropped) + case i: IncorrectRev => logger.warn(i)("An incorrect revision has been provided").as(ImportStatus.Dropped) + case other => IO.raiseError(other) + }, + _ => IO.pure(ImportStatus.Success) + ) + +} + +object ResourceProcessor { + + private val logger = Logger[ResourceProcessor] + + def apply( + log: ResourceLog, + fetchContext: FetchContext, + clock: EventClock + )(implicit jsonLdApi: JsonLdApi): ResourceProcessor = { + val rcr = ResolverContextResolution.never // TODO: Pass correct ResolverContextResolution + val resources = ResourcesImpl(log, fetchContext, rcr) + new ResourceProcessor(resources, clock) + } + +} diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resources/ResourceWiring.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resources/ResourceWiring.scala new file mode 100644 index 0000000000..52f4ba533c --- /dev/null +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resources/ResourceWiring.scala @@ -0,0 +1,41 @@ +package ch.epfl.bluebrain.nexus.ship.resources + +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.RemoteContextResolution +import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck +import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext +import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResourceResolution +import ch.epfl.bluebrain.nexus.delta.sdk.resources.Resources.ResourceLog +import ch.epfl.bluebrain.nexus.delta.sdk.resources.{DetectChange, FetchResource, Resources, ValidateResource} +import ch.epfl.bluebrain.nexus.delta.sdk.schemas.FetchSchema +import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig +import ch.epfl.bluebrain.nexus.delta.sourcing.{ScopedEventLog, Transactors} +import ch.epfl.bluebrain.nexus.ship.EventClock +import ch.epfl.bluebrain.nexus.ship.acls.AclWiring +import ch.epfl.bluebrain.nexus.ship.resolvers.ResolverWiring + +object ResourceWiring { + + def apply( + fetchContext: FetchContext, + fetchSchema: FetchSchema, + config: EventLogConfig, + clock: EventClock, + xas: Transactors + )(implicit + jsonLdApi: JsonLdApi + ): (ResourceLog, FetchResource) = { + val rcr = RemoteContextResolution.never // TODO: Use correct RemoteContextResolution + val detectChange = DetectChange(false) + val aclCheck = AclCheck(AclWiring.acls(config, clock, xas)) + val resolvers = ResolverWiring.resolvers(fetchContext, config, clock, xas) + val resourceResolution = + ResourceResolution.schemaResource(aclCheck, resolvers, fetchSchema, excludeDeprecated = false) + val validate = ValidateResource(resourceResolution)(rcr) + val resourceDef = Resources.definition(validate, detectChange, clock) + + val log = ScopedEventLog(resourceDef, config, xas) + (log, FetchResource(log)) + } + +} diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/schemas/SchemaProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/schemas/SchemaProcessor.scala new file mode 100644 index 0000000000..74e9ebf395 --- /dev/null +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/schemas/SchemaProcessor.scala @@ -0,0 +1,73 @@ +package ch.epfl.bluebrain.nexus.ship.schemas + +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.kernel.Logger +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext +import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution +import ch.epfl.bluebrain.nexus.delta.sdk.schemas.Schemas.SchemaLog +import ch.epfl.bluebrain.nexus.delta.sdk.schemas.model.SchemaEvent +import ch.epfl.bluebrain.nexus.delta.sdk.schemas.model.SchemaEvent._ +import ch.epfl.bluebrain.nexus.delta.sdk.schemas.model.SchemaRejection.{IncorrectRev, ResourceAlreadyExists} +import ch.epfl.bluebrain.nexus.delta.sdk.schemas.{SchemaImports, Schemas, SchemasImpl} +import ch.epfl.bluebrain.nexus.delta.sourcing.model.EntityType +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject +import ch.epfl.bluebrain.nexus.ship.schemas.SchemaProcessor.logger +import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, FailingUUID, ImportStatus} +import io.circe.Decoder + +class SchemaProcessor private (schemas: Schemas, clock: EventClock) extends EventProcessor[SchemaEvent] { + + override def resourceType: EntityType = Schemas.entityType + + override def decoder: Decoder[SchemaEvent] = SchemaEvent.serializer.codec + + override def evaluate(event: SchemaEvent): IO[ImportStatus] = { + for { + _ <- clock.setInstant(event.instant) + result <- evaluateInternal(event) + } yield result + } + + private def evaluateInternal(event: SchemaEvent): IO[ImportStatus] = { + implicit val s: Subject = event.subject + implicit val c: Caller = Caller(s, Set.empty) + val cRev = event.rev - 1 + + event match { + case e: SchemaCreated => schemas.create(e.id, e.project, e.source) + case e: SchemaUpdated => schemas.update(e.id, e.project, cRev, e.source) + case e: SchemaRefreshed => schemas.refresh(e.id, e.project) + case e: SchemaTagAdded => schemas.tag(e.id, e.project, e.tag, e.targetRev, cRev) + case e: SchemaTagDeleted => schemas.deleteTag(e.id, e.project, e.tag, cRev) + case e: SchemaDeprecated => schemas.deprecate(e.id, e.project, cRev) + case e: SchemaUndeprecated => schemas.undeprecate(e.id, e.project, cRev) + } + }.redeemWith( + { + case a: ResourceAlreadyExists => logger.warn(a)("The schema already exists").as(ImportStatus.Dropped) + case i: IncorrectRev => logger.warn(i)("An incorrect revision has been provided").as(ImportStatus.Dropped) + case other => IO.raiseError(other) + }, + _ => IO.pure(ImportStatus.Success) + ) + +} + +object SchemaProcessor { + + private val logger = Logger[SchemaProcessor] + + def apply( + log: SchemaLog, + fetchContext: FetchContext, + schemaImports: SchemaImports, + rcr: ResolverContextResolution, + clock: EventClock + )(implicit jsonLdApi: JsonLdApi): SchemaProcessor = { + val schemas = SchemasImpl(log, fetchContext, schemaImports, rcr)(jsonLdApi, FailingUUID) + new SchemaProcessor(schemas, clock) + } + +} diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/schemas/SchemaWiring.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/schemas/SchemaWiring.scala new file mode 100644 index 0000000000..ba0f0b2a79 --- /dev/null +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/schemas/SchemaWiring.scala @@ -0,0 +1,51 @@ +package ch.epfl.bluebrain.nexus.ship.schemas + +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi +import ch.epfl.bluebrain.nexus.delta.rdf.shacl.ShaclShapesGraph +import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck +import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext +import ch.epfl.bluebrain.nexus.delta.sdk.resources.FetchResource +import ch.epfl.bluebrain.nexus.delta.sdk.schemas.Schemas.SchemaLog +import ch.epfl.bluebrain.nexus.delta.sdk.schemas.{FetchSchema, SchemaImports, Schemas, ValidateSchema} +import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig +import ch.epfl.bluebrain.nexus.delta.sourcing.{ScopedEventLog, Transactors} +import ch.epfl.bluebrain.nexus.ship.acls.AclWiring +import ch.epfl.bluebrain.nexus.ship.resolvers.ResolverWiring +import ch.epfl.bluebrain.nexus.ship.{ContextWiring, EventClock} + +object SchemaWiring { + + def apply(config: EventLogConfig, clock: EventClock, xas: Transactors, api: JsonLdApi) = + for { + log <- schemaLog(config, clock, xas, api) + } yield (log, FetchSchema(log)) + + def schemaImports( + fetchResource: FetchResource, + fetchSchema: FetchSchema, + fetchContext: FetchContext, + config: EventLogConfig, + clock: EventClock, + xas: Transactors + )(implicit + jsonLdApi: JsonLdApi + ): SchemaImports = { + val aclCheck = AclCheck(AclWiring.acls(config, clock, xas)) + val resolvers = ResolverWiring.resolvers(fetchContext, config, clock, xas) + SchemaImports(aclCheck, resolvers, fetchSchema, fetchResource) + } + + private def validateSchema(implicit api: JsonLdApi): IO[ValidateSchema] = + for { + rcr <- ContextWiring.remoteContextResolution + shapesGraph <- ShaclShapesGraph.shaclShaclShapes + } yield ValidateSchema(api, shapesGraph, rcr) + + def schemaLog(config: EventLogConfig, clock: EventClock, xas: Transactors, api: JsonLdApi): IO[SchemaLog] = + for { + validate <- validateSchema(api) + schemaDef = Schemas.definition(validate, clock) + } yield ScopedEventLog(schemaDef, config, xas) + +} diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/BlazegraphViewProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/BlazegraphViewProcessor.scala new file mode 100644 index 0000000000..4714aba7fa --- /dev/null +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/BlazegraphViewProcessor.scala @@ -0,0 +1,88 @@ +package ch.epfl.bluebrain.nexus.ship.views + +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.kernel.Logger +import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF +import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.BlazegraphViewEvent._ +import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.BlazegraphViewRejection.{IncorrectRev, ResourceAlreadyExists} +import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.{BlazegraphViewEvent, BlazegraphViewValue, ViewResource} +import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.{BlazegraphViews, ValidateBlazegraphView} +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi +import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext +import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution +import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors +import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig +import ch.epfl.bluebrain.nexus.delta.sourcing.model.EntityType +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject +import ch.epfl.bluebrain.nexus.ship.views.BlazegraphViewProcessor.logger +import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, ImportStatus} +import io.circe.Decoder + +import java.util.UUID + +class BlazegraphViewProcessor private (views: UUID => IO[BlazegraphViews], clock: EventClock) + extends EventProcessor[BlazegraphViewEvent] { + + override def resourceType: EntityType = BlazegraphViews.entityType + + override def decoder: Decoder[BlazegraphViewEvent] = BlazegraphViewEvent.serializer.codec + + override def evaluate(event: BlazegraphViewEvent): IO[ImportStatus] = + for { + _ <- clock.setInstant(event.instant) + result <- evaluateInternal(event) + } yield result + + private def evaluateInternal(event: BlazegraphViewEvent): IO[ImportStatus] = { + implicit val s: Subject = event.subject + val cRev = event.rev - 1 + event match { + case e: BlazegraphViewCreated => views(event.uuid).flatMap(_.create(e.id, e.project, e.value)) + case e: BlazegraphViewUpdated => views(event.uuid).flatMap(_.update(e.id, e.project, cRev, e.value)) + case e: BlazegraphViewDeprecated => views(event.uuid).flatMap(_.deprecate(e.id, e.project, cRev)) + case e: BlazegraphViewUndeprecated => views(event.uuid).flatMap(_.undeprecate(e.id, e.project, cRev)) + case _: BlazegraphViewTagAdded => IO.unit // TODO: Can we tag? + } + }.redeemWith( + { + case a: ResourceAlreadyExists => logger.warn(a)("The resource already exists").as(ImportStatus.Dropped) + case i: IncorrectRev => logger.warn(i)("An incorrect revision has been provided").as(ImportStatus.Dropped) + case other => IO.raiseError(other) + }, + _ => IO.pure(ImportStatus.Success) + ) +} + +object BlazegraphViewProcessor { + + private val logger = Logger[BlazegraphViewProcessor] + + def apply( + fetchContext: FetchContext, + rcr: ResolverContextResolution, + config: EventLogConfig, + clock: EventClock, + xas: Transactors + )(implicit + jsonLdApi: JsonLdApi + ): BlazegraphViewProcessor = { + val noValidation = new ValidateBlazegraphView { + override def apply(value: BlazegraphViewValue): IO[Unit] = IO.unit + } + val prefix = "wrong_prefix" // TODO: fix prefix + + val views = (uuid: UUID) => + BlazegraphViews( + fetchContext, + rcr, + noValidation, + (_: ViewResource) => IO.unit, + config, + prefix, + xas, + clock + )(jsonLdApi, UUIDF.fixed(uuid)) + new BlazegraphViewProcessor(views, clock) + } + +} diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ElasticSearchViewProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ElasticSearchViewProcessor.scala new file mode 100644 index 0000000000..ab83d060d0 --- /dev/null +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/views/ElasticSearchViewProcessor.scala @@ -0,0 +1,96 @@ +package ch.epfl.bluebrain.nexus.ship.views + +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.kernel.Logger +import ch.epfl.bluebrain.nexus.delta.kernel.utils.{ClasspathResourceLoader, UUIDF} +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.ElasticSearchViewEvent._ +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.ElasticSearchViewRejection.{IncorrectRev, ResourceAlreadyExists} +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.{ElasticSearchFiles, ElasticSearchViewEvent, ElasticSearchViewValue} +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.{ElasticSearchViews, ValidateElasticSearchView} +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi +import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext +import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution +import ch.epfl.bluebrain.nexus.delta.sdk.views.IndexingRev +import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors +import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig +import ch.epfl.bluebrain.nexus.delta.sourcing.model.EntityType +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject +import ch.epfl.bluebrain.nexus.ship.views.ElasticSearchViewProcessor.logger +import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, ImportStatus} +import io.circe.Decoder + +import java.util.UUID + +class ElasticSearchViewProcessor private (views: UUID => IO[ElasticSearchViews], clock: EventClock) + extends EventProcessor[ElasticSearchViewEvent] { + + override def resourceType: EntityType = ElasticSearchViews.entityType + + override def decoder: Decoder[ElasticSearchViewEvent] = ElasticSearchViewEvent.serializer.codec + + override def evaluate(event: ElasticSearchViewEvent): IO[ImportStatus] = + for { + _ <- clock.setInstant(event.instant) + result <- evaluateInternal(event) + } yield result + + private def evaluateInternal(event: ElasticSearchViewEvent): IO[ImportStatus] = { + implicit val s: Subject = event.subject + val cRev = event.rev - 1 + event match { + case e: ElasticSearchViewCreated => views(event.uuid).flatMap(_.create(e.id, e.project, e.value)) + case e: ElasticSearchViewUpdated => views(event.uuid).flatMap(_.update(e.id, e.project, cRev, e.value)) + case e: ElasticSearchViewDeprecated => views(event.uuid).flatMap(_.deprecate(e.id, e.project, cRev)) + case e: ElasticSearchViewUndeprecated => views(event.uuid).flatMap(_.undeprecate(e.id, e.project, cRev)) + case _: ElasticSearchViewTagAdded => IO.unit // TODO: Check if this is correct + } + }.redeemWith( + { + case a: ResourceAlreadyExists => logger.warn(a)("The resource already exists").as(ImportStatus.Dropped) + case i: IncorrectRev => logger.warn(i)("An incorrect revision has been provided").as(ImportStatus.Dropped) + case other => IO.raiseError(other) + }, + _ => IO.pure(ImportStatus.Success) + ) + +} + +object ElasticSearchViewProcessor { + + private val logger = Logger[ElasticSearchViewProcessor] + + def apply( + fetchContext: FetchContext, + rcr: ResolverContextResolution, + config: EventLogConfig, + clock: EventClock, + xas: Transactors + )(implicit + jsonLdApi: JsonLdApi + ): IO[ElasticSearchViewProcessor] = { + implicit val loader: ClasspathResourceLoader = ClasspathResourceLoader.withContext(getClass) + + val noValidation = new ValidateElasticSearchView { + override def apply(uuid: UUID, indexingRev: IndexingRev, v: ElasticSearchViewValue): IO[Unit] = IO.unit + } + val prefix = "wrong_prefix" // TODO: fix prefix + val esFiles = ElasticSearchFiles.mk(loader) + + for { + files <- esFiles + views = (uuid: UUID) => + ElasticSearchViews( + fetchContext, + rcr, + noValidation, + config, + prefix, + xas, + files.defaultMapping, + files.defaultSettings, + clock + )(jsonLdApi, UUIDF.fixed(uuid)) + } yield new ElasticSearchViewProcessor(views, clock) + } + +} diff --git a/ship/src/test/resources/default.conf b/ship/src/test/resources/default.conf new file mode 100644 index 0000000000..fba48d6ebe --- /dev/null +++ b/ship/src/test/resources/default.conf @@ -0,0 +1,55 @@ +ship { + base-uri = "http://localhost:8080/v1" + + database { + read = ${ship.database.access} + # Access to database for write access + write = ${ship.database.access} + # Access to database for streaming access (indexing / SSEs) + streaming = ${ship.database.access} + + # when true it creates the tables on service boot + tables-autocreate = false + + cache { + # The max number of tokens in the partition cache + max-size = 1000 + # The duration after an entry in the cache expires + expire-after = 10 minutes + } + + access { + # the database host + host = 127.0.0.1 + # the database port + port = 5432 + # the pool size + pool-size = 10 + } + + name = "postgres" + username = "postgres" + password = "postgres" + } + + event-log { + query-config = { + batch-size = 30 + refresh-strategy = 3s + } + max-duration = 14 seconds + } + + organizations { + values { + # organization example + #obp = "The Open Brain Platform Organization" + } + } + + # Service account configuration for internal operations + service-account { + subject: "delta" + realm: "internal" + } +} \ No newline at end of file diff --git a/ship/src/test/resources/import/import.json b/ship/src/test/resources/import/import.json index 88f755d36c..b3d6cf7c17 100644 --- a/ship/src/test/resources/import/import.json +++ b/ship/src/test/resources/import/import.json @@ -4,6 +4,7 @@ {"ordering":2173453,"type":"resolver","org":"public","project":"sscx","id":"https://bluebrain.github.io/nexus/vocabulary/crossProject1","rev":2,"value":{"id": "https://bluebrain.github.io/nexus/vocabulary/crossProject1", "rev": 2, "tpe": "CrossProject", "@type": "ResolverDeprecated", "instant": "2020-01-28T08:23:36.270Z", "project": "public/sscx", "subject": {"@type": "User", "realm": "bbp", "subject": "bob"}},"instant":"2020-01-28T09:23:36.27+01:00"} {"ordering":2408475,"type":"resolver","org":"public","project":"sscx","id":"https://bluebrain.github.io/nexus/vocabulary/crossProject2","rev":1,"value":{"id": "https://bluebrain.github.io/nexus/vocabulary/crossProject2", "rev": 1, "@type": "ResolverCreated", "value": {"@type": "CrossProjectValue", "priority": 50, "projects": ["neurosciencegraph/datamodels"], "resourceTypes": [], "identityResolution": {"@type": "ProvidedIdentities", "value": [{"@type": "Authenticated", "realm": "bbp"}]}}, "source": {"@id": "https://bluebrain.github.io/nexus/vocabulary/crossProject2", "@type": ["CrossProject", "Resolver"], "@context": "https://bluebrain.github.io/nexus/contexts/resolvers.json", "priority": 50, "projects": ["neurosciencegraph/datamodels"], "identities": [{"@id": "https://bbp.epfl.ch/nexus/v1/realms/bbp/authenticated", "@type": "Authenticated", "realm": "bbp"}]}, "instant": "2020-03-09T13:36:19.246Z", "project": "public/sscx", "subject": {"@type": "User", "realm": "bbp", "subject": "bob"}},"instant":"2020-03-09T14:36:19.246+01:00"} {"ordering":4879496,"type":"resolver","org":"public","project":"sscx","id":"https://bluebrain.github.io/nexus/vocabulary/crossProject2","rev":2,"value":{"id": "https://bluebrain.github.io/nexus/vocabulary/crossProject2", "rev": 2, "@type": "ResolverUpdated", "value": {"@type": "CrossProjectValue", "priority": 50, "projects": ["neurosciencegraph/datamodels"], "resourceTypes": [], "identityResolution": {"@type": "UseCurrentCaller"}}, "source": {"@id": "https://bluebrain.github.io/nexus/vocabulary/crossProject2", "@type": ["Resolver", "CrossProject"], "@context": ["https://bluebrain.github.io/nexus/contexts/resolvers.json", "https://bluebrain.github.io/nexus/contexts/metadata.json"], "priority": 50, "projects": ["neurosciencegraph/datamodels"], "resourceTypes": [], "useCurrentCaller": true}, "instant": "2022-11-16T13:42:07.498Z", "project": "public/sscx", "subject": {"@type": "User", "realm": "bbp", "subject": "bob"}},"instant":"2022-11-16T14:42:07.498+01:00"} +{"ordering":4900000,"type":"resource","org":"public","project":"sscx","id":"https://bluebrain.github.io/nexus/vocabulary/crossProject2","rev":1,"value":{"id":"https://staging.nise.bbp.epfl.ch/nexus/v1/resources/tests/oliver/_/6285253a-0ed4-4d98-8821-fe030c3fef40","rev":1,"@type":"ResourceCreated","types":[],"schema":"https://bluebrain.github.io/nexus/schemas/unconstrained.json?rev=1","source":{"hello":"world"},"instant":"2024-03-15T17:20:08.294513Z","project":"public/sscx","subject":{"@type":"User","realm":"bbp","subject":"grabinsk"},"expanded":[{"@id":"https://staging.nise.bbp.epfl.ch/nexus/v1/resources/tests/oliver/_/6285253a-0ed4-4d98-8821-fe030c3fef40","https://staging.nise.bbp.epfl.ch/nexus/v1/vocabs/tests/oliver/hello":[{"@value":"world"}]}],"compacted":{"@id":"6285253a-0ed4-4d98-8821-fe030c3fef40","hello":"world","@context":{"@base":"https://staging.nise.bbp.epfl.ch/nexus/v1/resources/tests/oliver/_/","@vocab":"https://staging.nise.bbp.epfl.ch/nexus/v1/vocabs/tests/oliver/"}},"schemaProject":"tests/oliver","remoteContexts":[]},"instant":"2099-12-30T23:59:59.999+01:00"} {"ordering":5300965,"type":"project","org":"public","project":"sscx","id":"projects/public/sscx","rev":2,"value":{"rev": 2, "base": "https://bbp.epfl.ch/neurosciencegraph/data/", "uuid": "c7d70522-4305-480a-b190-75d757ed9a49", "@type": "ProjectUpdated", "label": "sscx", "vocab": "https://bbp.epfl.ch/ontologies/core/bmo/", "instant": "2023-07-16T18:42:59.530Z", "subject": {"@type": "User", "realm": "bbp", "subject": "alice"}, "apiMappings": {"prov": "http://www.w3.org/ns/prov#", "context": "https://incf.github.io/neuroshapes/contexts/", "schemaorg": "http://schema.org/", "datashapes": "https://neuroshapes.org/dash/", "ontologies": "https://neuroshapes.org/dash/ontology", "taxonomies": "https://neuroshapes.org/dash/taxonomy", "commonshapes": "https://neuroshapes.org/commons/", "provdatashapes": "https://provshapes.org/datashapes/", "provcommonshapes": "https://provshapes.org/commons/"}, "description": "This project contains somatosensorycortex dissemination publication data.", "organizationUuid": "d098a020-508b-4131-a28d-75f73b7f5f0e", "organizationLabel": "public"},"instant":"2023-07-16T20:42:59.53+02:00"} {"ordering":5318566,"type":"project","org":"public","project":"sscx","id":"projects/public/sscx","rev":3,"value":{"rev": 3, "base": "https://bbp.epfl.ch/nexus/v1/resources/public/sscx/_/", "uuid": "c7d70522-4305-480a-b190-75d757ed9a49", "@type": "ProjectUpdated", "label": "sscx", "vocab": "https://bbp.epfl.ch/ontologies/core/bmo/", "instant": "2023-07-21T13:55:02.463Z", "subject": {"@type": "User", "realm": "bbp", "subject": "alice"}, "apiMappings": {"prov": "http://www.w3.org/ns/prov#", "context": "https://incf.github.io/neuroshapes/contexts/", "schemaorg": "http://schema.org/", "datashapes": "https://neuroshapes.org/dash/", "ontologies": "https://neuroshapes.org/dash/ontology", "taxonomies": "https://neuroshapes.org/dash/taxonomy", "commonshapes": "https://neuroshapes.org/commons/", "provdatashapes": "https://provshapes.org/datashapes/", "provcommonshapes": "https://provshapes.org/commons/"}, "description": "This project contains somatosensorycortex dissemination publication data.", "organizationUuid": "d098a020-508b-4131-a28d-75f73b7f5f0e", "organizationLabel": "public"},"instant":"2023-07-21T15:55:02.463+02:00"} {"ordering":5418473,"type":"project","org":"public","project":"sscx","id":"projects/public/sscx","rev":4,"value":{"rev": 4, "base": "https://bbp.epfl.ch/data/public/sscx/", "uuid": "c7d70522-4305-480a-b190-75d757ed9a49", "@type": "ProjectUpdated", "label": "sscx", "vocab": "https://bbp.epfl.ch/ontologies/core/bmo/", "instant": "2023-08-22T15:05:13.654Z", "subject": {"@type": "User", "realm": "bbp", "subject": "alice"}, "apiMappings": {"prov": "http://www.w3.org/ns/prov#", "context": "https://incf.github.io/neuroshapes/contexts/", "schemaorg": "http://schema.org/", "datashapes": "https://neuroshapes.org/dash/", "ontologies": "https://neuroshapes.org/dash/ontology", "taxonomies": "https://neuroshapes.org/dash/taxonomy", "commonshapes": "https://neuroshapes.org/commons/", "provdatashapes": "https://provshapes.org/datashapes/", "provcommonshapes": "https://provshapes.org/commons/"}, "description": "This project contains somatosensorycortex dissemination publication data.", "organizationUuid": "d098a020-508b-4131-a28d-75f73b7f5f0e", "organizationLabel": "public"},"instant":"2023-08-22T17:05:13.654+02:00"} diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/EndToEndTest.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/EndToEndTest.scala new file mode 100644 index 0000000000..b6ee9b2877 --- /dev/null +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/EndToEndTest.scala @@ -0,0 +1,397 @@ +package ch.epfl.bluebrain.nexus.ship + +import akka.http.scaladsl.model.StatusCodes +import cats.data.NonEmptyList +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.kernel.utils.UrlUtils +import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri +import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv +import ch.epfl.bluebrain.nexus.delta.sourcing.exporter.ExportEventQuery +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Label, ProjectRef} +import ch.epfl.bluebrain.nexus.delta.sourcing.offset.Offset +import ch.epfl.bluebrain.nexus.tests.BaseIntegrationSpec +import ch.epfl.bluebrain.nexus.tests.Identity.writer +import ch.epfl.bluebrain.nexus.tests.admin.ProjectPayload +import ch.epfl.bluebrain.nexus.tests.iam.types.Permission +import ch.epfl.bluebrain.nexus.tests.iam.types.Permission.Export +import io.circe.Json +import io.circe.syntax.EncoderOps +import org.scalatest.Assertion + +import java.nio.file.{Files, Paths} +import scala.concurrent.duration.DurationInt +import scala.jdk.CollectionConverters.IteratorHasAsScala + +class EndToEndTest extends BaseIntegrationSpec { + + override def beforeAll(): Unit = { + super.beforeAll() + aclDsl.addPermission(s"/", writer, Export.Run).accepted + () + } + + "The ship" should { + + "transfer a project" in { + + val (project, _, projectJson) = thereIsAProject() + + whenTheExportIsRunOnProject(project) + + theOldProjectIsDeleted(project) + + weRunTheImporter(project) + + weFixThePermissions(project) + + thereShouldBeAProject(project, projectJson) + } + + "transfer multiple revisions of a project" in { + + val (project, revisionsAndStates) = thereAreManyRevisionsOfAProject() + + whenTheExportIsRunOnProject(project) + + theOldProjectIsDeleted(project, revisionsAndStates.keys.max) + + weRunTheImporter(project) + + weFixThePermissions(project) + + thereShouldBeAProjectThatMatchesExpectations(project, revisionsAndStates) + } + + "transfer the default resolver" in { + val (project, _, _) = thereIsAProject() + val defaultInProjectResolver = nxv + "defaultInProject" + val (_, resolverJson) = thereIsAResolver(defaultInProjectResolver, project) + + whenTheExportIsRunOnProject(project) + theOldProjectIsDeleted(project) + + weRunTheImporter(project) + weFixThePermissions(project) + + thereShouldBeAResolver(project, defaultInProjectResolver, resolverJson) + } + + "transfer a generic resource" in { + val (project, _, _) = thereIsAProject() + val (resource, resourceJson) = thereIsAResource(project) + + whenTheExportIsRunOnProject(project) + theOldProjectIsDeleted(project) + + weRunTheImporter(project) + weFixThePermissions(project) + + thereShouldBeAResource(project, resource, resourceJson) + } + + "transfer a schema" in { + val (project, _, _) = thereIsAProject() + val (schema, schemaJson) = thereIsASchema(project) + + whenTheExportIsRunOnProject(project) + theOldProjectIsDeleted(project) + + weRunTheImporter(project) + weFixThePermissions(project) + + thereShouldBeASchema(project, schema, schemaJson) + } + + "transfer an elasticsearch view" in { + val (project, _, _) = thereIsAProject() + val (esView, esViewJson) = thereIsAnElasticSearchView(project) + + whenTheExportIsRunOnProject(project) + theOldProjectIsDeleted(project) + + weRunTheImporter(project) + weFixThePermissions(project) + + thereShouldBeAView(project, esView, esViewJson) + } + + "transfer an blazegraph view" in { + val (project, _, _) = thereIsAProject() + val (bgView, bgViewJson) = thereIsABlazegraphView(project) + + whenTheExportIsRunOnProject(project) + theOldProjectIsDeleted(project) + + weRunTheImporter(project) + weFixThePermissions(project) + + thereShouldBeAView(project, bgView, bgViewJson) + } + + def thereShouldBeAView(project: ProjectRef, schema: Iri, originalJson: Json): Assertion = { + val encodedIri = UrlUtils.encode(schema.toString) + deltaClient + .get[Json](s"/views/${project.organization}/${project.project}/$encodedIri", writer) { (json, response) => + { + response.status shouldEqual StatusCodes.OK + json shouldEqual originalJson + } + } + .accepted + } + + def thereIsABlazegraphView(project: ProjectRef): (Iri, Json) = { + val simpleBgView = json"""{ + "@type": "SparqlView", + "includeMetadata": true, + "includeDeprecated": false, + "resourceTag": "mytag" + }""" + thereIsAView(project, simpleBgView) + } + + def thereIsAView(project: ProjectRef, body: Json): (Iri, Json) = { + val view = nxv + genString() + val encodedView = UrlUtils.encode(view.toString) + deltaClient + .put[Json](s"/views/${project.organization}/${project.project}/$encodedView", body, writer) { (_, response) => + response.status shouldEqual StatusCodes.Created + } + .accepted + + val (viewJson, status) = deltaClient + .getJsonAndStatus(s"/views/${project.organization}/${project.project}/$encodedView", writer) + .accepted + status shouldEqual StatusCodes.OK + view -> viewJson + } + + def thereIsAnElasticSearchView(project: ProjectRef): (Iri, Json) = { + val simpleEsView = + json""" + { + "@type": "ElasticSearchView", + "resourceSchemas": [], + "resourceTypes": [], + "sourceAsText": false, + "includeMetadata": true, + "includeDeprecated": false, + "mapping": {} + } + """ + thereIsAView(project, simpleEsView) + } + + def thereAreManyRevisionsOfAProject(): (ProjectRef, Map[Int, Json]) = { + val (ref, payload, json) = thereIsAProject() + val updatedJson = projectIsUpdated(ref, payload.copy(description = "updated description"), 1) + val deprecatedJson = projectIsDeprecated(ref, 2) + val undeprecatedJson = projectIsUndeprecated(ref, 3) + ( + ref, + Map( + 1 -> json, + 2 -> updatedJson, + 3 -> deprecatedJson, + 4 -> undeprecatedJson + ) + ) + } + + def thereIsAProject(): (ProjectRef, ProjectPayload, Json) = { + val orgName = genString() + val projName = genString() + + createOrg(writer, orgName).accepted + + val payload = ProjectPayload.generate(s"$orgName/$projName") + adminDsl.createProject(orgName, projName, payload, writer).accepted + + val ref = ProjectRef.unsafe(orgName, projName) + + (ref, payload, fetchProjectState(ref)) + } + + def fetchProjectState(project: ProjectRef) = { + val (projectJson, status) = + deltaClient.getJsonAndStatus(s"/projects/${project.organization}/${project.project}", writer).accepted + status shouldEqual StatusCodes.OK + projectJson + } + + def projectIsUpdated(ref: ProjectRef, projectPayload: ProjectPayload, revision: Int): Json = { + adminDsl.updateProject(ref.organization.value, ref.project.value, projectPayload, writer, revision).accepted + fetchProjectState(ref) + } + + def projectIsDeprecated(ref: ProjectRef, rev: Int): Json = { + val (_, statusCode) = + deltaClient.deleteJsonAndStatus(s"/projects/${ref.organization}/${ref.project}?rev=$rev", writer).accepted + statusCode shouldBe StatusCodes.OK + fetchProjectState(ref) + } + + def projectIsUndeprecated(ref: ProjectRef, rev: Int): Json = { + val (_, statusCode) = deltaClient + .putJsonAndStatus(s"/projects/${ref.organization}/${ref.project}/undeprecate?rev=$rev", Json.obj(), writer) + .accepted + statusCode shouldBe StatusCodes.OK + fetchProjectState(ref) + } + + def whenTheExportIsRunOnProject(project: ProjectRef): Unit = { + val query = ExportEventQuery( + Label.unsafe(project.project.value), + NonEmptyList.of(project), + Offset.start + ).asJson + + deltaClient + .post[Json]("/export/events", query, writer) { (_, response) => + response.status shouldEqual StatusCodes.Accepted + } + .accepted + + IO.sleep(6.seconds).accepted + } + + def theOldProjectIsDeleted(project: ProjectRef, rev: Int = 1): Unit = { + deltaClient + .delete[Json](s"/projects/${project.organization}/${project.project}?rev=$rev&prune=true", writer) { + (_, response) => response.status shouldEqual StatusCodes.OK + } + .accepted + + eventually { + deltaClient.get[Json](s"/projects/${project.organization}/${project.project}", writer) { (_, response) => + response.status shouldEqual StatusCodes.NotFound + } + } + () + } + + def weRunTheImporter(project: ProjectRef): Unit = { + val folder = s"/tmp/ship/${project.project.value}/" + val folderPath = Paths.get(folder) + val file = Files.newDirectoryStream(folderPath, "*.json").iterator().asScala.toList.head + + new RunShip().run(fs2.io.file.Path.fromNioPath(file), None).accepted + () + } + + def thereShouldBeAProject(project: ProjectRef, originalJson: Json): Assertion = { + deltaClient + .get[Json](s"/projects/${project.organization}/${project.project}", writer) { (json, response) => + { + response.status shouldEqual StatusCodes.OK + json shouldEqual originalJson + } + } + .accepted + } + + def thereShouldBeAProjectThatMatchesExpectations(project: ProjectRef, expectations: Map[Int, Json]): Assertion = { + expectations.foreach { case (rev, expectedJson) => + thereShouldBeAProjectRevision(project, rev, expectedJson) + } + succeed + } + + def weFixThePermissions(project: ProjectRef) = + aclDsl.addPermissions(s"/$project", writer, Permission.minimalPermissions).accepted + + def thereIsAResolver(resolver: Iri, project: ProjectRef): (Iri, Json) = { + val encodedResolver = UrlUtils.encode(resolver.toString) + val (resolverJson, status) = deltaClient + .getJsonAndStatus(s"/resolvers/${project.organization}/${project.project}/$encodedResolver", writer) + .accepted + status shouldEqual StatusCodes.OK + resolver -> resolverJson + } + + def thereShouldBeAResolver(project: ProjectRef, resolver: Iri, originalJson: Json): Assertion = { + val encodedResolver = UrlUtils.encode(resolver.toString) + deltaClient + .get[Json](s"/resolvers/${project.organization}/${project.project}/$encodedResolver", writer) { + (json, response) => + { + response.status shouldEqual StatusCodes.OK + json shouldEqual originalJson + } + } + .accepted + } + + def thereIsAResource(project: ProjectRef): (Iri, Json) = { + val resource = nxv + genString() + val encodedResource = UrlUtils.encode(resource.toString) + val body = json"""{"hello": "world"}""" + deltaClient + .put[Json](s"/resources/${project.organization}/${project.project}/_/$encodedResource", body, writer) { + (_, response) => + response.status shouldEqual StatusCodes.Created + } + .accepted + val (resourceJson, status) = deltaClient + .getJsonAndStatus(s"/resources/${project.organization}/${project.project}/_/$encodedResource", writer) + .accepted + status shouldEqual StatusCodes.OK + resource -> resourceJson + } + + def thereShouldBeAResource(project: ProjectRef, resource: Iri, originalJson: Json): Assertion = { + val encodedResolver = UrlUtils.encode(resource.toString) + deltaClient + .get[Json](s"/resources/${project.organization}/${project.project}/_/$encodedResolver", writer) { + (json, response) => + { + response.status shouldEqual StatusCodes.OK + json shouldEqual originalJson + } + } + .accepted + } + + def thereIsASchema(project: ProjectRef): (Iri, Json) = { + val schema = nxv + genString() + val encodedSchema = UrlUtils.encode(schema.toString) + // TODO: Review the json of the simpleSchema + val simpleSchema = + json"""{"shapes":[{"@id":"http://example.com/MyShape","@type":"http://www.w3.org/ns/shacl#NodeShape","nodeKind":"http://www.w3.org/ns/shacl#BlankNodeOrIRI","targetClass":"http://example.com/Custom","property":[{"path":"http://example.com/name","datatype":"http://www.w3.org/2001/XMLSchema#string","minCount":1}]}]}""" + deltaClient + .put[Json](s"/schemas/${project.organization}/${project.project}/$encodedSchema", simpleSchema, writer) { + (_, response) => + response.status shouldEqual StatusCodes.Created + } + .accepted + val (resourceJson, status) = deltaClient + .getJsonAndStatus(s"/schemas/${project.organization}/${project.project}/$encodedSchema", writer) + .accepted + status shouldEqual StatusCodes.OK + schema -> resourceJson + } + + def thereShouldBeASchema(project: ProjectRef, schema: Iri, originalJson: Json): Assertion = { + val encodedIri = UrlUtils.encode(schema.toString) + deltaClient + .get[Json](s"/schemas/${project.organization}/${project.project}/$encodedIri", writer) { (json, response) => + { + response.status shouldEqual StatusCodes.OK + json shouldEqual originalJson + } + } + .accepted + } + def thereShouldBeAProjectRevision(project: ProjectRef, rev: Int, expectedProjectJson: Json): Assertion = { + deltaClient + .get[Json](s"/projects/${project.organization}/${project.project}?rev=$rev", writer) { (json, response) => + { + response.status shouldEqual StatusCodes.OK + json shouldEqual expectedProjectJson + } + } + .accepted + } + } + +} diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/MainSuite.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/MainSuite.scala index 1bbcf4f93c..f2dcc8279e 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/MainSuite.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/MainSuite.scala @@ -5,6 +5,7 @@ import cats.syntax.all._ import ch.epfl.bluebrain.nexus.delta.kernel.utils.ClasspathResourceLoader import ch.epfl.bluebrain.nexus.delta.sdk.projects.Projects import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.Resolvers +import ch.epfl.bluebrain.nexus.delta.sdk.resources.Resources import ch.epfl.bluebrain.nexus.delta.sourcing.model.EntityType import ch.epfl.bluebrain.nexus.delta.sourcing.offset.Offset import ch.epfl.bluebrain.nexus.delta.sourcing.postgres.Doobie.{PostgresPassword, PostgresUser} @@ -29,12 +30,13 @@ class MainSuite extends NexusSuite with MainSuite.Fixture { Map( Projects.entityType -> Count(5L, 0L), Resolvers.entityType -> Count(5L, 0L), + Resources.entityType -> Count(1L, 0L), EntityType("xxx") -> Count(0L, 1L) ) ) for { importFile <- ClasspathResourceLoader().absolutePath("import/import.json").map(Path(_)) - _ <- Main.run(importFile, None).assertEquals(expected) + _ <- new RunShip().run(importFile, None).assertEquals(expected) } yield () } diff --git a/tests/docker/docker-compose.yml b/tests/docker/docker-compose.yml index 32161cf1a4..0107119452 100644 --- a/tests/docker/docker-compose.yml +++ b/tests/docker/docker-compose.yml @@ -36,6 +36,7 @@ services: volumes: - ./config:/config - /tmp:/default-volume + - /tmp/ship:/tmp extra_hosts: - "delta:127.0.0.1" dns: @@ -138,6 +139,8 @@ services: postgres: image: library/postgres:15.6 + ports: + - 5432:5432 environment: POSTGRES_USER: "postgres" POSTGRES_PASSWORD: "postgres" diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/BaseIntegrationSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/BaseIntegrationSpec.scala index c6e972f3e6..eff92acff2 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/BaseIntegrationSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/BaseIntegrationSpec.scala @@ -64,7 +64,7 @@ trait BaseIntegrationSpec val deltaUrl: Uri = Uri(s"http://${sys.props.getOrElse("delta-url", "localhost:8080")}/v1") - private[tests] val deltaClient = HttpClient(deltaUrl) + val deltaClient: HttpClient = HttpClient(deltaUrl) val elasticsearchDsl = new ElasticsearchDsl() val blazegraphDsl = new BlazegraphDsl() diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala index 76e8d5a29c..b08ad47df6 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala @@ -64,6 +64,12 @@ class HttpClient private (baseUrl: Uri, httpExt: HttpExt)(implicit body.flatMap(body => requestAssert(POST, url, Some(body), identity, extraHeaders)(assertResponse)) } + def putJsonAndStatus(url: String, body: Json, identity: Identity)(implicit + um: FromEntityUnmarshaller[Json] + ): IO[(Json, StatusCode)] = { + requestJsonAndStatus(PUT, url, Some(body), identity, jsonHeaders) + } + def put[A](url: String, body: Json, identity: Identity, extraHeaders: Seq[HttpHeader] = jsonHeaders)( assertResponse: (A, HttpResponse) => Assertion )(implicit um: FromEntityUnmarshaller[A]): IO[Assertion] = @@ -200,6 +206,18 @@ class HttpClient private (baseUrl: Uri, httpExt: HttpExt)(implicit requestJson(GET, url, None, identity, (a: A, _: HttpResponse) => a, jsonHeaders) } + def getJsonAndStatus(url: String, identity: Identity)(implicit + um: FromEntityUnmarshaller[Json] + ): IO[(Json, StatusCode)] = { + requestJsonAndStatus(GET, url, None, identity, jsonHeaders) + } + + def deleteJsonAndStatus(url: String, identity: Identity)(implicit + um: FromEntityUnmarshaller[Json] + ): IO[(Json, StatusCode)] = { + requestJsonAndStatus(DELETE, url, None, identity, jsonHeaders) + } + def delete[A](url: String, identity: Identity, extraHeaders: Seq[HttpHeader] = jsonHeaders)( assertResponse: (A, HttpResponse) => Assertion )(implicit um: FromEntityUnmarshaller[A]): IO[Assertion] = @@ -261,6 +279,23 @@ class HttpClient private (baseUrl: Uri, httpExt: HttpExt)(implicit ) } + def requestJsonAndStatus( + method: HttpMethod, + url: String, + body: Option[Json], + identity: Identity, + extraHeaders: Seq[HttpHeader] + )(implicit um: FromEntityUnmarshaller[Json]): IO[(Json, StatusCode)] = + request[Json, Json, (Json, StatusCode)]( + method, + url, + body, + identity, + (j: Json) => HttpEntity(ContentTypes.`application/json`, j.noSpaces), + (json, response) => (json, response.status), + extraHeaders + ) + def requestJson[A, R]( method: HttpMethod, url: String, diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/Identity.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/Identity.scala index 9325ae5990..a8dce65e92 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/Identity.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/Identity.scala @@ -36,6 +36,8 @@ object Identity extends Generators { // User with an invalid token val InvalidTokenUser: UserCredentials = UserCredentials(genString(), genString(), testRealm) + val writer = UserCredentials(genString(), genString(), testRealm) + object acls { val Marge = UserCredentials(genString(), genString(), testRealm) } @@ -107,6 +109,6 @@ object Identity extends Generators { } lazy val allUsers = - userPermissions.UserWithNoPermissions :: userPermissions.UserWithPermissions :: acls.Marge :: archives.Tweety :: compositeviews.Jerry :: events.BugsBunny :: listings.Bob :: listings.Alice :: aggregations.Charlie :: aggregations.Rose :: orgs.Fry :: orgs.Leela :: projects.Bojack :: projects.PrincessCarolyn :: resources.Rick :: resources.Morty :: storages.Coyote :: views.ScoobyDoo :: mash.Radar :: supervision.Mickey :: files.Writer :: typehierarchy.Writer :: Nil + userPermissions.UserWithNoPermissions :: userPermissions.UserWithPermissions :: acls.Marge :: archives.Tweety :: compositeviews.Jerry :: events.BugsBunny :: listings.Bob :: listings.Alice :: aggregations.Charlie :: aggregations.Rose :: orgs.Fry :: orgs.Leela :: projects.Bojack :: projects.PrincessCarolyn :: resources.Rick :: resources.Morty :: storages.Coyote :: views.ScoobyDoo :: mash.Radar :: supervision.Mickey :: files.Writer :: typehierarchy.Writer :: writer :: Nil }