Merge branch 'master' into refactor-s3-storage-access

delta/plugins/elasticsearch/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/client/ElasticSearchClientSpec.scala

@@ -4,6 +4,7 @@ import akka.actor.ActorSystem
 import akka.http.scaladsl.model.StatusCodes
 import akka.http.scaladsl.model.Uri.Query
 import akka.testkit.TestKit
+import cats.effect.IO
 import cats.implicits._
 import ch.epfl.bluebrain.nexus.delta.kernel.search.Pagination.FromPagination
 import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.ScalaTestElasticSearchClientSetup
@@ -23,7 +24,7 @@ import ch.epfl.bluebrain.nexus.testkit.CirceLiteral
 import ch.epfl.bluebrain.nexus.testkit.elasticsearch.ElasticSearchDocker
 import ch.epfl.bluebrain.nexus.testkit.scalatest.ce.CatsEffectSpec
 import io.circe.{Json, JsonObject}
-import org.scalatest.DoNotDiscover
+import org.scalatest.{Assertion, DoNotDiscover}
 import org.scalatest.concurrent.Eventually
 
 import scala.concurrent.duration._
@@ -260,23 +261,25 @@ class ElasticSearchClientSpec(override val docker: ElasticSearchDocker)
         ElasticSearchAction.Update(index, "1", json"""{ "doc" : {"field2" : "value2"} }""")
       )
 
+      def theCountShouldBe(count: Long): IO[Assertion] =
+        esClient.count(index.value).map(_ shouldEqual count)
+
       {
         for {
           // Indexing and checking count
-          _        <- esClient.bulk(operations)
-          _        <- esClient.refresh(index)
-          original <- esClient.count(index.value)
-          _         = original shouldEqual 2L
+          _    <- esClient.bulk(operations)
+          _    <- esClient.refresh(index)
+          _    <- eventually { theCountShouldBe(2L) }
           // Deleting document matching the given query
-          query     = jobj"""{"query": {"bool": {"must": {"term": {"field1": 3} } } } }"""
-          _        <- esClient.deleteByQuery(query, index)
+          query = jobj"""{"query": {"bool": {"must": {"term": {"field1": 3} } } } }"""
+          _    <- esClient.deleteByQuery(query, index)
           // Checking docs again
-          newCount <- esClient.count(index.value)
-          _         = newCount shouldEqual 1L
-          doc1     <- esClient.getSource[Json](index, "1").attemptNarrow[HttpClientError]
-          _         = doc1.rightValue
-          doc2     <- esClient.getSource[Json](index, "2").attemptNarrow[HttpClientError]
-          _         = doc2.leftValue.errorCode.value shouldEqual StatusCodes.NotFound
+          _    <- esClient.refresh(index)
+          _    <- eventually { theCountShouldBe(1L) }
+          doc1 <- esClient.getSource[Json](index, "1").attemptNarrow[HttpClientError]
+          _     = doc1.rightValue
+          doc2 <- esClient.getSource[Json](index, "2").attemptNarrow[HttpClientError]
+          _     = doc2.leftValue.errorCode.value shouldEqual StatusCodes.NotFound
         } yield ()
       }.accepted
     }

delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resolvers/ResolversImpl.scala

@@ -165,7 +165,12 @@ object ResolversImpl {
       uuidF: UUIDF
   ): Resolvers = {
     def priorityAlreadyExists(ref: ProjectRef, self: Iri, priority: Priority): IO[Unit] = {
-      sql"SELECT id FROM scoped_states WHERE type = ${Resolvers.entityType} AND org = ${ref.organization} AND project = ${ref.project}  AND id != $self AND (value->'value'->'priority')::int = ${priority.value} "
+      sql"""SELECT id FROM scoped_states
+            WHERE type = ${Resolvers.entityType}
+            AND org = ${ref.organization} AND project = ${ref.project}
+            AND id != $self
+            AND (value->'deprecated')::boolean = false
+            AND (value->'value'->'priority')::int = ${priority.value}"""
         .query[Iri]
         .option
         .transact(xas.read)

delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/resolvers/ResolversImplSpec.scala

@@ -8,7 +8,6 @@ import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF
 import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.{contexts, nxv, schema}
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.{JsonLdApi, JsonLdJavaApi}
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.RemoteContextResolution
-import ch.epfl.bluebrain.nexus.delta.sdk.ConfigFixtures
 import ch.epfl.bluebrain.nexus.delta.sdk.generators.ProjectGen
 import ch.epfl.bluebrain.nexus.delta.sdk.generators.ResolverGen.{resolverResourceFor, sourceFrom, sourceWithoutId}
 import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller
@@ -24,6 +23,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.model.ResolverRejection.{Inco
 import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.model.ResolverValue.{CrossProjectValue, InProjectValue}
 import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.model._
 import ch.epfl.bluebrain.nexus.delta.sdk.resources.Resources
+import ch.epfl.bluebrain.nexus.delta.sdk.{ConfigFixtures, ResolverResource}
 import ch.epfl.bluebrain.nexus.delta.sourcing.EntityDependencyStore
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.EntityDependency.DependsOn
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.{Authenticated, Group, User}
@@ -620,6 +620,22 @@ class ResolversImplSpec extends CatsEffectSpec with DoobieScalaTestFixture with
       }
 
     }
+
+    "validating priority" should {
+      "allow creating a resolver with the same priority as a deprecated resolver" in {
+        givenADeprecatedInProjectResolver(666) { resolver =>
+          resolvers.create(nxv + genString(), resolver.value.project, InProjectValue(resolver.value.priority))
+        }
+      }
+    }
+
+    def givenADeprecatedInProjectResolver(priority: Int)(test: ResolverResource => IO[ResolverResource]) = {
+      val id                 = nxv + genString()
+      val resolverValue      = InProjectValue(Priority.unsafe(priority))
+      resolvers.create(id, projectRef, resolverValue).accepted
+      val deprecatedResolver = resolvers.deprecate(id, projectRef, 1).accepted
+      test(deprecatedResolver).accepted
+    }
   }
 
 }

delta/sourcing-psql/src/main/scala/ch/epfl/bluebrain/nexus/delta/sourcing/Transactors.scala

@@ -69,7 +69,7 @@ object Transactors {
   /**
     * For testing purposes, drop the current tables and then executes the different available scripts
     */
-  private[sourcing] def dropAndCreateDDLs: IO[List[String]] = ddls.map(dropScript :: _)
+  def dropAndCreateDDLs: IO[List[String]] = ddls.map(dropScript :: _)
 
   /** Type of a cache that contains the hashed names of the projectRefs for which a partition was already created. */
   type PartitionsCache = LocalCache[String, Unit]

delta/sourcing-psql/src/test/scala/ch/epfl/bluebrain/nexus/delta/sourcing/postgres/Doobie.scala

@@ -17,7 +17,7 @@ object Doobie {
   val PostgresUser     = "postgres"
   val PostgresPassword = "postgres"
 
-  private def transactors(
+  def transactors(
       postgres: Resource[IO, PostgresContainer],
       user: String,
       pass: String

ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/ContextWiring.scala

@@ -2,22 +2,20 @@ package ch.epfl.bluebrain.nexus.ship
 
 import cats.effect.IO
 import ch.epfl.bluebrain.nexus.delta.kernel.utils.ClasspathResourceLoader
+import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.{contexts => bgContexts}
+import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.{contexts => compositeViewContexts}
+import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.{contexts => esContexts}
 import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.contexts
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution}
-import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck
 import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext
 import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution
 import ch.epfl.bluebrain.nexus.delta.sdk.resources.FetchResource
 import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors
 import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig
-import ch.epfl.bluebrain.nexus.ship.acls.AclWiring
+import ch.epfl.bluebrain.nexus.ship.acls.AclWiring.alwaysAuthorize
 import ch.epfl.bluebrain.nexus.ship.resolvers.ResolverWiring
 
-import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.{contexts => esContexts}
-import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.model.{contexts => bgContexts}
-import ch.epfl.bluebrain.nexus.delta.plugins.compositeviews.model.{contexts => compositeViewContexts}
-
 object ContextWiring {
 
   implicit private val loader: ClasspathResourceLoader = ClasspathResourceLoader.withContext(getClass)
@@ -53,12 +51,11 @@ object ContextWiring {
       clock: EventClock,
       xas: Transactors
   )(implicit jsonLdApi: JsonLdApi): IO[ResolverContextResolution] = {
-    val aclCheck  = AclCheck(AclWiring.acls(config, clock, xas))
     val resolvers = ResolverWiring.resolvers(fetchContext, config, clock, xas)
 
     for {
       rcr <- remoteContextResolution
-    } yield ResolverContextResolution(aclCheck, resolvers, rcr, fetchResource)
+    } yield ResolverContextResolution(alwaysAuthorize, resolvers, rcr, fetchResource)
   }
 
 }

ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/Main.scala

@@ -41,8 +41,8 @@ object Main
   override def main: Opts[IO[ExitCode]] =
     (run orElse showConfig)
       .map {
-        case Run(file, config, _) => new RunShip().run(file, config)
-        case ShowConfig(config)   => showConfig(config)
+        case Run(file, config, offset) => new RunShip().run(file, config, offset)
+        case ShowConfig(config)        => showConfig(config)
       }
       .map(_.as(ExitCode.Success))
 

ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala

@@ -9,6 +9,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext
 import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ApiMappings
 import ch.epfl.bluebrain.nexus.delta.sdk.quotas.Quotas
 import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors
+import ch.epfl.bluebrain.nexus.delta.sourcing.offset.Offset
 import ch.epfl.bluebrain.nexus.ship.config.ShipConfig
 import ch.epfl.bluebrain.nexus.ship.model.InputEvent
 import ch.epfl.bluebrain.nexus.ship.organizations.OrganizationProvider
@@ -25,7 +26,7 @@ class RunShip {
 
   private val logger = Logger[RunShip]
 
-  def run(file: Path, config: Option[Path]): IO[ImportReport] = {
+  def run(file: Path, config: Option[Path], fromOffset: Offset = Offset.start): IO[ImportReport] = {
     val clock                         = Clock[IO]
     val uuidF                         = UUIDF.random
     // Resources may have been created with different configurations so we adopt the lenient one for the import
@@ -42,7 +43,7 @@ class RunShip {
                   for {
                     // Provision organizations
                     _                           <- orgProvider.create(config.organizations.values)
-                    events                       = eventStream(file)
+                    events                       = eventStream(file, fromOffset)
                     fetchActiveOrg               = FetchActiveOrganization(xas)
                     // Wiring
                     eventClock                  <- EventClock.init()
@@ -83,11 +84,17 @@ class RunShip {
     } yield report
   }
 
-  private def eventStream(file: Path): Stream[IO, InputEvent] =
-    Files[IO].readUtf8Lines(file).zipWithIndex.evalMap { case (line, index) =>
-      IO.fromEither(decode[InputEvent](line)).onError { err =>
-        logger.error(err)(s"Error parsing to event at line $index")
+  private def eventStream(file: Path, fromOffset: Offset): Stream[IO, InputEvent] =
+    Files[IO]
+      .readUtf8Lines(file)
+      .zipWithIndex
+      .evalMap { case (line, index) =>
+        IO.fromEither(decode[InputEvent](line)).onError { err =>
+          logger.error(err)(s"Error parsing to event at line $index")
+        }
+      }
+      .filter { event =>
+        event.ordering.value >= fromOffset.value
       }
-    }
 
 }

ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/acls/AclWiring.scala

@@ -1,23 +1,44 @@
 package ch.epfl.bluebrain.nexus.ship.acls
 
-import cats.effect.{Clock, IO}
-import ch.epfl.bluebrain.nexus.delta.sdk.acls.{Acls, AclsImpl}
+import cats.effect.IO
+import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck
+import ch.epfl.bluebrain.nexus.delta.sdk.acls.model.AclAddress
+import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller
 import ch.epfl.bluebrain.nexus.delta.sdk.permissions.model.Permission
-import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors
-import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig
+import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity
+
+import scala.collection.immutable
 
 object AclWiring {
 
-  def acls(config: EventLogConfig, clock: Clock[IO], xas: Transactors): Acls = {
-    val permissionSet = Set(Permission.unsafe("resources/read"))
-    AclsImpl(
-      IO.pure(permissionSet),
-      AclsImpl.findUnknownRealms(xas),
-      permissionSet,
-      config,
-      xas,
-      clock
-    )
+  def alwaysAuthorize: AclCheck = new AclCheck {
+    override def authorizeForOr[E <: Throwable](path: AclAddress, permission: Permission, identities: Set[Identity])(
+        onError: => E
+    ): IO[Unit] = IO.unit
+
+    override def authorizeFor(path: AclAddress, permission: Permission, identities: Set[Identity]): IO[Boolean] =
+      IO.pure(true)
+
+    override def authorizeForEveryOr[E <: Throwable](path: AclAddress, permissions: Set[Permission])(onError: => E)(
+        implicit caller: Caller
+    ): IO[Unit] = IO.unit
+
+    override def mapFilterOrRaise[A, B](
+        values: immutable.Iterable[A],
+        extractAddressPermission: A => (AclAddress, Permission),
+        onAuthorized: A => B,
+        onFailure: AclAddress => IO[Unit]
+    )(implicit caller: Caller): IO[Set[B]] =
+      IO.pure(values.map(onAuthorized).toSet)
+
+    override def mapFilterAtAddressOrRaise[A, B](
+        values: immutable.Iterable[A],
+        address: AclAddress,
+        extractPermission: A => Permission,
+        onAuthorized: A => B,
+        onFailure: AclAddress => IO[Unit]
+    )(implicit caller: Caller): IO[Set[B]] =
+      IO.pure(values.map(onAuthorized).toSet)
   }
 
 }

ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/resources/ResourceWiring.scala

@@ -2,7 +2,6 @@ package ch.epfl.bluebrain.nexus.ship.resources
 
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.RemoteContextResolution
-import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck
 import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext
 import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResourceResolution
 import ch.epfl.bluebrain.nexus.delta.sdk.resources.Resources.ResourceLog
@@ -11,7 +10,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.schemas.FetchSchema
 import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig
 import ch.epfl.bluebrain.nexus.delta.sourcing.{ScopedEventLog, Transactors}
 import ch.epfl.bluebrain.nexus.ship.EventClock
-import ch.epfl.bluebrain.nexus.ship.acls.AclWiring
+import ch.epfl.bluebrain.nexus.ship.acls.AclWiring.alwaysAuthorize
 import ch.epfl.bluebrain.nexus.ship.resolvers.ResolverWiring
 
 object ResourceWiring {
@@ -27,10 +26,9 @@ object ResourceWiring {
   ): (ResourceLog, FetchResource) = {
     val rcr                = RemoteContextResolution.never // TODO: Use correct RemoteContextResolution
     val detectChange       = DetectChange(false)
-    val aclCheck           = AclCheck(AclWiring.acls(config, clock, xas))
     val resolvers          = ResolverWiring.resolvers(fetchContext, config, clock, xas)
     val resourceResolution =
-      ResourceResolution.schemaResource(aclCheck, resolvers, fetchSchema, excludeDeprecated = false)
+      ResourceResolution.schemaResource(alwaysAuthorize, resolvers, fetchSchema, excludeDeprecated = false)
     val validate           = ValidateResource(resourceResolution)(rcr)
     val resourceDef        = Resources.definition(validate, detectChange, clock)
 

ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/schemas/SchemaWiring.scala

@@ -3,14 +3,13 @@ package ch.epfl.bluebrain.nexus.ship.schemas
 import cats.effect.IO
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi
 import ch.epfl.bluebrain.nexus.delta.rdf.shacl.ShaclShapesGraph
-import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck
 import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext
 import ch.epfl.bluebrain.nexus.delta.sdk.resources.FetchResource
 import ch.epfl.bluebrain.nexus.delta.sdk.schemas.Schemas.SchemaLog
 import ch.epfl.bluebrain.nexus.delta.sdk.schemas.{FetchSchema, SchemaImports, Schemas, ValidateSchema}
 import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig
 import ch.epfl.bluebrain.nexus.delta.sourcing.{ScopedEventLog, Transactors}
-import ch.epfl.bluebrain.nexus.ship.acls.AclWiring
+import ch.epfl.bluebrain.nexus.ship.acls.AclWiring.alwaysAuthorize
 import ch.epfl.bluebrain.nexus.ship.resolvers.ResolverWiring
 import ch.epfl.bluebrain.nexus.ship.{ContextWiring, EventClock}
 
@@ -31,9 +30,8 @@ object SchemaWiring {
   )(implicit
       jsonLdApi: JsonLdApi
   ): SchemaImports = {
-    val aclCheck  = AclCheck(AclWiring.acls(config, clock, xas))
     val resolvers = ResolverWiring.resolvers(fetchContext, config, clock, xas)
-    SchemaImports(aclCheck, resolvers, fetchSchema, fetchResource)
+    SchemaImports(alwaysAuthorize, resolvers, fetchSchema, fetchResource)
   }
 
   private def validateSchema(implicit api: JsonLdApi): IO[ValidateSchema] =

ship/src/test/resources/import/two-projects.json

@@ -0,0 +1,2 @@
+{"ordering":1,"type":"project","org":"public","project":"sscx","id":"projects/public/sscx","rev":1,"value":{"rev": 1, "base": "https://bbp.epfl.ch/neurosciencegraph/data/", "uuid": "c7d70522-4305-480a-b190-75d757ed9a49", "@type": "ProjectCreated", "label": "sscx", "vocab": "https://bbp.epfl.ch/nexus/v1/resources/public/sscx/_/", "instant": "2020-01-22T16:03:33.105Z", "subject": {"@type": "User", "realm": "bbp", "subject": "bob"}, "apiMappings": {"prov": "http://www.w3.org/ns/prov#", "context": "https://incf.github.io/neuroshapes/contexts/", "schemaorg": "http://schema.org/", "datashapes": "https://neuroshapes.org/dash/", "ontologies": "https://neuroshapes.org/dash/ontology", "taxonomies": "https://neuroshapes.org/dash/taxonomy", "commonshapes": "https://neuroshapes.org/commons/", "provdatashapes": "https://provshapes.org/datashapes/", "provcommonshapes": "https://provshapes.org/commons/"}, "description": "This project contains somatosensorycortex dissemination publication data.", "organizationUuid": "d098a020-508b-4131-a28d-75f73b7f5f0e", "organizationLabel": "public"},"instant":"2020-01-22T17:03:33.105+01:00"}
+{"ordering":2,"type":"project","org":"public","project":"other","id":"projects/public/other","rev":1,"value":{"rev": 1, "base": "https://bbp.epfl.ch/neurosciencegraph/data/", "uuid": "c7d70522-4305-480a-b190-75d757ed9a49", "@type": "ProjectCreated", "label": "other", "vocab": "https://bbp.epfl.ch/nexus/v1/resources/public/other/_/", "instant": "2020-01-22T16:03:33.105Z", "subject": {"@type": "User", "realm": "bbp", "subject": "bob"}, "apiMappings": {"prov": "http://www.w3.org/ns/prov#", "context": "https://incf.github.io/neuroshapes/contexts/", "schemaorg": "http://schema.org/", "datashapes": "https://neuroshapes.org/dash/", "ontologies": "https://neuroshapes.org/dash/ontology", "taxonomies": "https://neuroshapes.org/dash/taxonomy", "commonshapes": "https://neuroshapes.org/commons/", "provdatashapes": "https://provshapes.org/datashapes/", "provcommonshapes": "https://provshapes.org/commons/"}, "description": "This is a second project.", "organizationUuid": "d098a020-508b-4131-a28d-75f73b7f5f0e", "organizationLabel": "public"},"instant":"2020-01-22T17:03:33.105+01:00"}