Merge branch 'master' into refactor-s3-storage-access

BlueBrain · Mar 26, 2024 · 8a8e91e · 8a8e91e
2 parents 918bea5 + f296a28
commit 8a8e91e
Show file tree

Hide file tree

Showing 61 changed files with 1,755 additions and 344 deletions.
diff --git a/.github/workflows/ci-delta-ship.yml b/.github/workflows/ci-delta-ship.yml
@@ -23,15 +23,27 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - name: Create /tmp/ship folder
+        run: mkdir -p /tmp/ship
       - name: Setup JDK
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: '21'
           cache: 'sbt'
           check-latest: true
-      - name: Unit tests
+      - name: Clean, build Delta & Storage images
         run: |
           sbt -Dsbt.color=always -Dsbt.supershell=false \
             clean \
-            ship-unit-tests-with-coverage
+            app/Docker/publishLocal
+      - name: Start services
+        run: docker-compose -f tests/docker/docker-compose.yml up -d
+      - name: Waiting for Delta to start
+        run: |
+          URL="http://localhost:8080/v1/version"
+          curl --connect-timeout 3 --max-time 5 --retry 30 --retry-all-errors --retry-delay 3 --retry-max-time 90 $URL
+      - name: Unit tests
+        run: |
+          sbt -Dsbt.color=always -Dsbt.supershell=false \
+            ship-unit-tests
diff --git a/build.sbt b/build.sbt
@@ -741,7 +741,12 @@ lazy val ship = project
   )
   .enablePlugins(UniversalPlugin, JavaAppPackaging, JavaAgent, DockerPlugin, BuildInfoPlugin)
   .settings(shared, compilation, servicePackaging, assertJavaVersion, kamonSettings, coverage, release)
-  .dependsOn(sdk % "compile->compile;test->test", testkit % "test->compile")
+  .dependsOn(
+    sdk                 % "compile->compile;test->test",
+    blazegraphPlugin    % "compile->compile",
+    elasticsearchPlugin % "compile->compile",
+    tests               % "test->compile;test->test"
+  )
   .settings(
     libraryDependencies ++= Seq(declineEffect),
     addCompilerPlugin(betterMonadicFor),

diff --git a/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/AclsModule.scala b/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/AclsModule.scala
@@ -37,7 +37,7 @@ object AclsModule extends ModuleDef {
         permissions.fetchPermissionSet,
         AclsImpl.findUnknownRealms(xas),
         permissions.minimum,
-        config.acls,
+        config.acls.eventLog,
         xas,
         clock
       )

diff --git a/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/ResourcesModule.scala b/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/ResourcesModule.scala
@@ -19,18 +19,23 @@ import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext
 import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ApiMappings
 import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverResolution.ResourceResolution
 import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.{ResolverContextResolution, Resolvers, ResourceResolution}
+import ch.epfl.bluebrain.nexus.delta.sdk.resources.Resources.{ResourceDefinition, ResourceLog}
+import ch.epfl.bluebrain.nexus.delta.sdk.resources._
 import ch.epfl.bluebrain.nexus.delta.sdk.resources.model.{Resource, ResourceEvent}
-import ch.epfl.bluebrain.nexus.delta.sdk.resources.{DetectChange, Resources, ResourcesConfig, ResourcesImpl, ValidateResource}
-import ch.epfl.bluebrain.nexus.delta.sdk.schemas.Schemas
+import ch.epfl.bluebrain.nexus.delta.sdk.schemas.FetchSchema
 import ch.epfl.bluebrain.nexus.delta.sdk.schemas.model.Schema
 import ch.epfl.bluebrain.nexus.delta.sdk.sse.SseEncoder
-import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors
+import ch.epfl.bluebrain.nexus.delta.sourcing.{ScopedEventLog, Transactors}
 import izumi.distage.model.definition.{Id, ModuleDef}
 
 /**
   * Resources wiring
   */
 object ResourcesModule extends ModuleDef {
+  make[ResourceResolution[Schema]].from { (aclCheck: AclCheck, resolvers: Resolvers, fetchSchema: FetchSchema) =>
+    ResourceResolution.schemaResource(aclCheck, resolvers, fetchSchema, excludeDeprecated = false)
+  }
+
   make[ValidateResource].from {
     (resourceResolution: ResourceResolution[Schema], rcr: RemoteContextResolution @Id("aggregate")) =>
       ValidateResource(resourceResolution)(rcr)
@@ -40,39 +45,44 @@ object ResourcesModule extends ModuleDef {
 
   make[DetectChange].from { (config: ResourcesConfig) => DetectChange(config.skipUpdateNoChange) }
 
+  make[ResourceDefinition].from { (validateResource: ValidateResource, detectChange: DetectChange, clock: Clock[IO]) =>
+    Resources.definition(validateResource, detectChange, clock)
+  }
+
+  make[ResourceLog].from { (scopedDefinition: ResourceDefinition, config: ResourcesConfig, xas: Transactors) =>
+    ScopedEventLog(scopedDefinition, config.eventLog, xas)
+  }
+
+  make[FetchResource].from { (scopedLog: ResourceLog) =>
+    FetchResource(scopedLog)
+  }
+
   make[Resources].from {
     (
-        validate: ValidateResource,
-        detectChange: DetectChange,
+        resourceLog: ResourceLog,
         fetchContext: FetchContext,
-        config: ResourcesConfig,
         resolverContextResolution: ResolverContextResolution,
         api: JsonLdApi,
-        xas: Transactors,
-        clock: Clock[IO],
         uuidF: UUIDF
     ) =>
       ResourcesImpl(
-        validate,
-        detectChange,
+        resourceLog,
         fetchContext,
-        resolverContextResolution,
-        config,
-        xas,
-        clock
+        resolverContextResolution
       )(
         api,
         uuidF
       )
   }
 
   make[ResolverContextResolution].from {
-    (aclCheck: AclCheck, resolvers: Resolvers, resources: Resources, rcr: RemoteContextResolution @Id("aggregate")) =>
-      ResolverContextResolution(aclCheck, resolvers, resources, rcr)
-  }
-
-  make[ResourceResolution[Schema]].from { (aclCheck: AclCheck, resolvers: Resolvers, schemas: Schemas) =>
-    ResourceResolution.schemaResource(aclCheck, resolvers, schemas, excludeDeprecated = false)
+    (
+        aclCheck: AclCheck,
+        resolvers: Resolvers,
+        rcr: RemoteContextResolution @Id("aggregate"),
+        fetchResource: FetchResource
+    ) =>
+      ResolverContextResolution(aclCheck, resolvers, rcr, fetchResource)
   }
 
   make[ResourcesRoutes].from {

diff --git a/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/SchemasModule.scala b/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/SchemasModule.scala
@@ -21,11 +21,12 @@ import ch.epfl.bluebrain.nexus.delta.sdk.model.metrics.ScopedEventMetricEncoder
 import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext
 import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ApiMappings
 import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.{ResolverContextResolution, Resolvers}
-import ch.epfl.bluebrain.nexus.delta.sdk.resources.Resources
+import ch.epfl.bluebrain.nexus.delta.sdk.resources.FetchResource
+import ch.epfl.bluebrain.nexus.delta.sdk.schemas.Schemas.{SchemaDefinition, SchemaLog}
+import ch.epfl.bluebrain.nexus.delta.sdk.schemas._
 import ch.epfl.bluebrain.nexus.delta.sdk.schemas.model.{Schema, SchemaEvent}
-import ch.epfl.bluebrain.nexus.delta.sdk.schemas.{SchemaImports, Schemas, SchemasImpl, ValidateSchema}
 import ch.epfl.bluebrain.nexus.delta.sdk.sse.SseEncoder
-import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors
+import ch.epfl.bluebrain.nexus.delta.sourcing.{ScopedEventLog, Transactors}
 import izumi.distage.model.definition.{Id, ModuleDef}
 
 /**
@@ -40,37 +41,43 @@ object SchemasModule extends ModuleDef {
 
   }
 
+  make[SchemaDefinition].from { (validateSchema: ValidateSchema, clock: Clock[IO]) =>
+    Schemas.definition(validateSchema, clock)
+  }
+
+  make[SchemaLog].from { (scopedDefinition: SchemaDefinition, config: AppConfig, xas: Transactors) =>
+    ScopedEventLog(scopedDefinition, config.schemas.eventLog, xas)
+  }
+
+  make[FetchSchema].from { (schemaLog: SchemaLog) =>
+    FetchSchema(schemaLog)
+  }
+
   make[Schemas].from {
     (
+        schemaLog: SchemaLog,
         fetchContext: FetchContext,
         schemaImports: SchemaImports,
         api: JsonLdApi,
-        validate: ValidateSchema,
         resolverContextResolution: ResolverContextResolution,
-        config: AppConfig,
-        xas: Transactors,
-        clock: Clock[IO],
         uuidF: UUIDF
     ) =>
       SchemasImpl(
+        schemaLog,
         fetchContext,
         schemaImports,
-        resolverContextResolution,
-        validate,
-        config.schemas,
-        xas,
-        clock
+        resolverContextResolution
       )(api, uuidF)
   }
 
   make[SchemaImports].from {
     (
         aclCheck: AclCheck,
         resolvers: Resolvers,
-        resources: Resources,
-        schemas: Schemas
+        fetchSchema: FetchSchema,
+        fetchResource: FetchResource
     ) =>
-      SchemaImports(aclCheck, resolvers, schemas, resources)
+      SchemaImports(aclCheck, resolvers, fetchSchema, fetchResource)
   }
 
   make[SchemasRoutes].from {

diff --git a/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/MainSuite.scala b/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/MainSuite.scala
@@ -17,6 +17,7 @@ import munit.catseffect.IOFixture
 import munit.{AnyFixture, CatsEffectSuite}
 
 import java.nio.file.{Files, Paths}
+import scala.concurrent.duration.Duration
 
 /**
   * Test class that allows to check that across core and plugins:
@@ -26,6 +27,8 @@ import java.nio.file.{Files, Paths}
   */
 class MainSuite extends NexusSuite with MainSuite.Fixture {
 
+  override val munitIOTimeout: Duration = Duration(60, "s")
+
   private val pluginsParentPath  = Paths.get("target/plugins").toAbsolutePath
   private val pluginLoaderConfig = PluginLoaderConfig(pluginsParentPath.toString)
 

diff --git a/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/AclsRoutesSpec.scala b/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/AclsRoutesSpec.scala
@@ -7,7 +7,7 @@ import cats.effect.IO
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords
 import ch.epfl.bluebrain.nexus.delta.sdk.acls.model.AclAddress.{Organization, Project, Root}
 import ch.epfl.bluebrain.nexus.delta.sdk.acls.model.{Acl, AclAddress}
-import ch.epfl.bluebrain.nexus.delta.sdk.acls.{AclCheck, Acls, AclsConfig, AclsImpl}
+import ch.epfl.bluebrain.nexus.delta.sdk.acls.{AclCheck, Acls, AclsImpl}
 import ch.epfl.bluebrain.nexus.delta.sdk.identities.IdentitiesDummy
 import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller
 import ch.epfl.bluebrain.nexus.delta.sdk.implicits._
@@ -90,7 +90,7 @@ class AclsRoutesSpec extends BaseRouteSpec {
       IO.pure(Set(aclsPermissions.read, aclsPermissions.write, managePermission, events.read)),
       Acls.findUnknownRealms(_, Set(realm1, realm2)),
       Set(aclsPermissions.read, aclsPermissions.write, managePermission, events.read),
-      AclsConfig(EventLogConfig(QueryConfig(5, RefreshStrategy.Stop), 100.millis)),
+      EventLogConfig(QueryConfig(5, RefreshStrategy.Stop), 100.millis),
       xas,
       clock
     )

diff --git a/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/ResourcesRoutesSpec.scala b/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/ResourcesRoutesSpec.scala
@@ -17,16 +17,17 @@ import ch.epfl.bluebrain.nexus.delta.sdk.generators.{ProjectGen, ResourceResolut
 import ch.epfl.bluebrain.nexus.delta.sdk.identities.IdentitiesDummy
 import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller
 import ch.epfl.bluebrain.nexus.delta.sdk.implicits._
+import ch.epfl.bluebrain.nexus.delta.sdk.model.Fetch.FetchF
 import ch.epfl.bluebrain.nexus.delta.sdk.model.{IdSegmentRef, ResourceUris}
 import ch.epfl.bluebrain.nexus.delta.sdk.permissions.Permissions.resources
 import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContextDummy
 import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ApiMappings
 import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution
-import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverResolution.FetchResource
 import ch.epfl.bluebrain.nexus.delta.sdk.resources.NexusSource.DecodingOption
-import ch.epfl.bluebrain.nexus.delta.sdk.resources.{DetectChange, Resources, ResourcesConfig, ResourcesImpl, ValidateResource}
+import ch.epfl.bluebrain.nexus.delta.sdk.resources._
 import ch.epfl.bluebrain.nexus.delta.sdk.schemas.model.Schema
 import ch.epfl.bluebrain.nexus.delta.sdk.utils.BaseRouteSpec
+import ch.epfl.bluebrain.nexus.delta.sourcing.ScopedEventLog
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.{Anonymous, Authenticated, Group, Subject, User}
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef}
@@ -87,7 +88,7 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
 
   private val aclCheck = AclSimpleCheck().accepted
 
-  private val fetchSchema: (ResourceRef, ProjectRef) => FetchResource[Schema] = {
+  private val fetchSchema: (ResourceRef, ProjectRef) => FetchF[Schema] = {
     case (ref, _) if ref.iri == schema2.id => IO.pure(Some(SchemaGen.resourceFor(schema2, deprecated = true)))
     case (ref, _) if ref.iri == schema1.id => IO.pure(Some(SchemaGen.resourceFor(schema1)))
     case (ref, _) if ref.iri == schema3.id => IO.pure(Some(SchemaGen.resourceFor(schema3)))
@@ -101,14 +102,17 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
   private val resolverContextResolution: ResolverContextResolution = ResolverContextResolution(rcr)
 
   private def routesWithDecodingOption(implicit decodingOption: DecodingOption): (Route, Resources) = {
+    val resourceDef = Resources.definition(validator, DetectChange(enabled = true), clock)
+    val scopedLog   = ScopedEventLog(
+      resourceDef,
+      ResourcesConfig(eventLogConfig, decodingOption, skipUpdateNoChange = true).eventLog,
+      xas
+    )
+
     val resources = ResourcesImpl(
-      validator,
-      DetectChange(enabled = true),
+      scopedLog,
       fetchContext,
-      resolverContextResolution,
-      ResourcesConfig(eventLogConfig, decodingOption, skipUpdateNoChange = true),
-      xas,
-      clock
+      resolverContextResolution
     )
     (
       Route.seal(

diff --git a/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/SchemasRoutesSpec.scala b/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/SchemasRoutesSpec.scala
@@ -23,8 +23,9 @@ import ch.epfl.bluebrain.nexus.delta.sdk.permissions.Permissions.schemas
 import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContextDummy
 import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ApiMappings
 import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution
-import ch.epfl.bluebrain.nexus.delta.sdk.schemas.{SchemaImports, SchemasConfig, SchemasImpl, ValidateSchema}
+import ch.epfl.bluebrain.nexus.delta.sdk.schemas.{SchemaImports, Schemas, SchemasConfig, SchemasImpl, ValidateSchema}
 import ch.epfl.bluebrain.nexus.delta.sdk.utils.BaseRouteSpec
+import ch.epfl.bluebrain.nexus.delta.sourcing.ScopedEventLog
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.{Anonymous, Authenticated, Group, Subject, User}
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef
 import ch.epfl.bluebrain.nexus.testkit.ce.IOFromMap
@@ -78,12 +79,15 @@ class SchemasRoutesSpec extends BaseRouteSpec with IOFromMap with CatsIOValues {
 
   private val config = SchemasConfig(eventLogConfig)
 
+  private val schemaDef      = Schemas.definition(ValidateSchema.apply, clock)
+  private lazy val schemaLog = ScopedEventLog(schemaDef, config.eventLog, xas)
+
   private lazy val routes =
     Route.seal(
       SchemasRoutes(
         identities,
         aclCheck,
-        SchemasImpl(fetchContext, schemaImports, resolverContextResolution, ValidateSchema.apply, config, xas, clock),
+        SchemasImpl(schemaLog, fetchContext, schemaImports, resolverContextResolution),
         groupDirectives,
         IndexingAction.noop
       )

diff --git a/...aph/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/blazegraph/BlazegraphViews.scala b/...aph/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/blazegraph/BlazegraphViews.scala
@@ -532,7 +532,7 @@ object BlazegraphViews {
     apply(fetchContext, contextResolution, validate, createNameSpace, eventLogConfig, prefix, xas, clock)
   }
 
-  private[blazegraph] def apply(
+  def apply(
       fetchContext: FetchContext,
       contextResolution: ResolverContextResolution,
       validate: ValidateBlazegraphView,

diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/acls/AclsImpl.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/acls/AclsImpl.scala
@@ -13,6 +13,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.permissions.model.Permission
 import ch.epfl.bluebrain.nexus.delta.sdk.realms.Realms
 import ch.epfl.bluebrain.nexus.delta.sdk.syntax._
 import ch.epfl.bluebrain.nexus.delta.sourcing._
+import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label
 import ch.epfl.bluebrain.nexus.delta.sourcing.state.GlobalStateStore
@@ -108,12 +109,12 @@ object AclsImpl {
       fetchPermissionSet: IO[Set[Permission]],
       findUnknownRealms: Set[Label] => IO[Unit],
       minimum: Set[Permission],
-      config: AclsConfig,
+      config: EventLogConfig,
       xas: Transactors,
       clock: Clock[IO]
   ): Acls =
     new AclsImpl(
-      GlobalEventLog(Acls.definition(fetchPermissionSet, findUnknownRealms, clock), config.eventLog, xas),
+      GlobalEventLog(Acls.definition(fetchPermissionSet, findUnknownRealms, clock), config, xas),
       minimum
     )
 

diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/Fetch.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/Fetch.scala
@@ -0,0 +1,10 @@
+package ch.epfl.bluebrain.nexus.delta.sdk.model
+
+import cats.effect.IO
+
+object Fetch {
+
+  type Fetch[R]  = IO[Option[R]]
+  type FetchF[R] = Fetch[ResourceF[R]]
+
+}