Merge branch 'master' into update-docker-compose

BlueBrain · Apr 3, 2024 · 863e7af · 863e7af
2 parents a67d4d6 + 54214d8
commit 863e7af
Show file tree

Hide file tree

Showing 25 changed files with 300 additions and 104 deletions.
diff --git a/...arch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/search/model/SearchConfig.scala b/...arch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/search/model/SearchConfig.scala
@@ -34,17 +34,6 @@ final case class SearchConfig(
 
 object SearchConfig {
 
-  implicit val projectRefReader: ConfigReader[ProjectRef] = ConfigReader.fromString { value =>
-    value.split("/").toList match {
-      case orgStr :: projectStr :: Nil =>
-        (Label(orgStr), Label(projectStr))
-          .mapN(ProjectRef(_, _))
-          .leftMap(err => CannotConvert(value, classOf[ProjectRef].getSimpleName, err.getMessage))
-      case _                           =>
-        Left(CannotConvert(value, classOf[ProjectRef].getSimpleName, "Wrong format"))
-    }
-  }
-
   type Suite  = Set[ProjectRef]
   type Suites = Map[Label, Suite]
 

diff --git a/...ourcing-psql/src/main/scala/ch/epfl/bluebrain/nexus/delta/sourcing/model/ProjectRef.scala b/...ourcing-psql/src/main/scala/ch/epfl/bluebrain/nexus/delta/sourcing/model/ProjectRef.scala
@@ -9,6 +9,8 @@ import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.decoder.JsonLdDecoderError.Parsi
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.encoder.JsonLdEncoder
 import doobie.{Get, Put}
 import io.circe.{Decoder, Encoder, KeyDecoder, KeyEncoder}
+import pureconfig.ConfigReader
+import pureconfig.error.CannotConvert
 
 /**
   * A project label along with its parent organization label.
@@ -70,4 +72,15 @@ object ProjectRef {
     (projectRef.organization.value, projectRef.project.value)
   }
 
+  implicit val projectRefConfigReader: ConfigReader[ProjectRef] = ConfigReader.fromString { value =>
+    value.split("/").toList match {
+      case orgStr :: projectStr :: Nil =>
+        (Label(orgStr), Label(projectStr))
+          .mapN(ProjectRef(_, _))
+          .leftMap(err => CannotConvert(value, classOf[ProjectRef].getSimpleName, err.getMessage))
+      case _                           =>
+        Left(CannotConvert(value, classOf[ProjectRef].getSimpleName, "Wrong format"))
+    }
+  }
+
 }
diff --git a/...ting-started/notebooks/MOOC_Content_based_Recommender_System_using_Blue_Brain_Nexus.ipynb b/...ting-started/notebooks/MOOC_Content_based_Recommender_System_using_Blue_Brain_Nexus.ipynb
@@ -144,7 +144,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "r = requests.get('https://raw.githubusercontent.com/BlueBrain/nexus/ef830192d4e7bb95f9351c4bdab7b0114c27e2f0/docs/src/main/paradox/docs/getting-started/notebooks/rdfmodel/jsonldcontext.json')\n",
+    "r = requests.get('https://raw.githubusercontent.com/BlueBrain/nexus/d9f6cb83a27149c29bc604f3c34ea00c9ad64e67/docs/src/main/paradox/docs/getting-started/notebooks/rdfmodel/jsonldcontext.json')\n",
     "dirpath = './rdfmodel'\n",
     "Path(dirpath).mkdir(parents=True, exist_ok=True)\n",
     "with open(f'{dirpath}/jsonldcontext.json', 'w') as outfile:\n",
@@ -168,7 +168,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "forge = KnowledgeGraphForge(\"https://raw.githubusercontent.com/BlueBrain/nexus/ef830192d4e7bb95f9351c4bdab7b0114c27e2f0/docs/src/main/paradox/docs/getting-started/notebooks/forge.yml\",\n",
+    "forge = KnowledgeGraphForge(\"https://raw.githubusercontent.com/BlueBrain/nexus/d9f6cb83a27149c29bc604f3c34ea00c9ad64e67/docs/src/main/paradox/docs/getting-started/notebooks/forge.yml\",\n",
     "                            bucket=f\"{ORG}/{PROJECT}\",\n",
     "                            endpoint=ENDPOINT,\n",
     "                            token=TOKEN)"

diff --git a/docs/src/main/paradox/docs/getting-started/notebooks/dataset_from_different_sources.ipynb b/docs/src/main/paradox/docs/getting-started/notebooks/dataset_from_different_sources.ipynb
@@ -136,7 +136,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "r = requests.get('https://raw.githubusercontent.com/BlueBrain/nexus/ef830192d4e7bb95f9351c4bdab7b0114c27e2f0/docs/src/main/paradox/docs/getting-started/notebooks/rdfmodel/jsonldcontext.json')\n",
+    "r = requests.get('https://raw.githubusercontent.com/BlueBrain/nexus/d9f6cb83a27149c29bc604f3c34ea00c9ad64e67/docs/src/main/paradox/docs/getting-started/notebooks/rdfmodel/jsonldcontext.json')\n",
     "dirpath = './rdfmodel'\n",
     "Path(dirpath).mkdir(parents=True, exist_ok=True)\n",
     "with open(f'{dirpath}/jsonldcontext.json', 'w') as outfile:\n",
@@ -159,7 +159,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "forge = KnowledgeGraphForge(\"https://raw.githubusercontent.com/BlueBrain/nexus/ef830192d4e7bb95f9351c4bdab7b0114c27e2f0/docs/src/main/paradox/docs/getting-started/notebooks/forge.yml\",\n",
+    "forge = KnowledgeGraphForge(\"https://raw.githubusercontent.com/BlueBrain/nexus/d9f6cb83a27149c29bc604f3c34ea00c9ad64e67/docs/src/main/paradox/docs/getting-started/notebooks/forge.yml\",\n",
     "                            bucket=f\"{ORG}/{PROJECT}\",\n",
     "                            endpoint=\"https://sandbox.bluebrainnexus.io/v1\",\n",
     "                            token=TOKEN)"
@@ -483,7 +483,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "allen_nm_mapping = DictionaryMapping.load(\"https://raw.githubusercontent.com/BlueBrain/nexus/ef830192d4e7bb95f9351c4bdab7b0114c27e2f0/docs/src/main/paradox/docs/getting-started/notebooks/mappings/allen_morphology_dataset.hjson\") # TODO\n",
+    "allen_nm_mapping = DictionaryMapping.load(\"https://raw.githubusercontent.com/BlueBrain/nexus/d9f6cb83a27149c29bc604f3c34ea00c9ad64e67/docs/src/main/paradox/docs/getting-started/notebooks/mappings/allen_morphology_dataset.hjson\") # TODO\n",
     "nm_allen_resources = forge.map(nm_allen_metadata, allen_nm_mapping)"
    ]
   },
@@ -501,7 +501,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "allen_ephys_mapping = DictionaryMapping.load(\"https://raw.githubusercontent.com/BlueBrain/nexus/ef830192d4e7bb95f9351c4bdab7b0114c27e2f0/docs/src/main/paradox/docs/getting-started/notebooks/mappings/allen_ephys_dataset.hjson\") # TODO\n",
+    "allen_ephys_mapping = DictionaryMapping.load(\"https://raw.githubusercontent.com/BlueBrain/nexus/d9f6cb83a27149c29bc604f3c34ea00c9ad64e67/docs/src/main/paradox/docs/getting-started/notebooks/mappings/allen_ephys_dataset.hjson\") # TODO\n",
     "nephys_allen_resources = forge.map(nm_allen_metadata, allen_ephys_mapping)"
    ]
   },
@@ -519,7 +519,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "mouselight_nm_mapping = DictionaryMapping.load(\"https://raw.githubusercontent.com/BlueBrain/nexus/ef830192d4e7bb95f9351c4bdab7b0114c27e2f0/docs/src/main/paradox/docs/getting-started/notebooks/mappings/mouselight_dataset.hjson\") # TODO\n",
+    "mouselight_nm_mapping = DictionaryMapping.load(\"https://raw.githubusercontent.com/BlueBrain/nexus/d9f6cb83a27149c29bc604f3c34ea00c9ad64e67/docs/src/main/paradox/docs/getting-started/notebooks/mappings/mouselight_dataset.hjson\") # TODO\n",
     "nm_mouselight_resources = forge.map(nm_mouselight_metadata, mouselight_nm_mapping)"
    ]
   },

diff --git a/docs/src/main/paradox/docs/getting-started/notebooks/file-to-resource-mapping.hjson b/docs/src/main/paradox/docs/getting-started/notebooks/file-to-resource-mapping.hjson
@@ -0,0 +1,25 @@
+{
+    type: DataDownload
+    contentSize:
+    {
+        unitCode: f"bytes"
+        value: x._bytes
+    }
+    digest:
+    {
+        algorithm: x._digest._algorithm
+        value: x._digest._value
+    }
+    encodingFormat: x._mediaType
+    name: x._filename
+    contentUrl: x._self
+    atLocation:
+    {
+        type: Location
+        store: 
+        {
+            id: x._storage["@id"]
+        }
+        location: x._location if '_location' in x else None
+    }
+}
diff --git a/docs/src/main/paradox/docs/getting-started/notebooks/forge.yml b/docs/src/main/paradox/docs/getting-started/notebooks/forge.yml
@@ -20,7 +20,7 @@ Store:
     project_property: "https://bluebrain.github.io/nexus/vocabulary/project"
   max_connection: 5
   versioned_id_template: "{x.id}?rev={x._store_metadata._rev}"
-  file_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-store/file-to-resource-mapping.hjson
+  file_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus/8851236f5bc38e113f1f04af2ca0bb31dab1c576/docs/src/main/paradox/docs/getting-started/notebooks/file-to-resource-mapping.hjson
 
 Formatters:
   identifier: https://bbp.epfl.ch/neurosciencegraph/data/{}/{}
diff --git a/docs/src/main/paradox/docs/getting-started/notebooks/one_cell_minds.ipynb b/docs/src/main/paradox/docs/getting-started/notebooks/one_cell_minds.ipynb
@@ -139,7 +139,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "r = requests.get('https://raw.githubusercontent.com/BlueBrain/nexus/ef830192d4e7bb95f9351c4bdab7b0114c27e2f0/docs/src/main/paradox/docs/getting-started/notebooks/rdfmodel/jsonldcontext.json')\n",
+    "r = requests.get('https://raw.githubusercontent.com/BlueBrain/nexus/d9f6cb83a27149c29bc604f3c34ea00c9ad64e67/docs/src/main/paradox/docs/getting-started/notebooks/rdfmodel/jsonldcontext.json')\n",
     "dirpath = './rdfmodel'\n",
     "Path(dirpath).mkdir(parents=True, exist_ok=True)\n",
     "with open(f'{dirpath}/jsonldcontext.json', 'w') as outfile:\n",
@@ -162,11 +162,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "forge = KnowledgeGraphForge(\"https://raw.githubusercontent.com/BlueBrain/nexus/ef830192d4e7bb95f9351c4bdab7b0114c27e2f0/docs/src/main/paradox/docs/getting-started/notebooks/forge.yml\",\n",
+    "forge = KnowledgeGraphForge(\"https://raw.githubusercontent.com/BlueBrain/nexus/d9f6cb83a27149c29bc604f3c34ea00c9ad64e67/docs/src/main/paradox/docs/getting-started/notebooks/forge.yml\",\n",
     "                            bucket=f\"{ORG}/{PROJECT}\",\n",
     "                            endpoint=\"https://sandbox.bluebrainnexus.io/v1\",\n",
-    "                            token=TOKEN,\n",
-    "                            file_resource_mapping='https://raw.githubusercontent.com/BlueBrain/nexus-forge/5580e906eb1df9e3b3970a735b82f609207df717/examples/configurations/nexus-store/file-to-resource-mapping.hjson'))"
+    "                            token=TOKEN)"
    ]
   },
   {
@@ -334,7 +333,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "allen_nm_mapping = DictionaryMapping.load(\"https://raw.githubusercontent.com/BlueBrain/nexus/ef830192d4e7bb95f9351c4bdab7b0114c27e2f0/docs/src/main/paradox/docs/getting-started/notebooks/mappings/allen_morphology_dataset.hjson\")\n",
+    "allen_nm_mapping = DictionaryMapping.load(\"https://raw.githubusercontent.com/BlueBrain/nexus/d9f6cb83a27149c29bc604f3c34ea00c9ad64e67/docs/src/main/paradox/docs/getting-started/notebooks/mappings/allen_morphology_dataset.hjson\")\n",
     "nm_allen_resources = forge.map(nm_allen_metadata, allen_nm_mapping, na='')"
    ]
   },
@@ -351,7 +350,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "allen_ephys_mapping = DictionaryMapping.load(\"https://raw.githubusercontent.com/BlueBrain/nexus/ef830192d4e7bb95f9351c4bdab7b0114c27e2f0/docs/src/main/paradox/docs/getting-started/notebooks/mappings/allen_ephys_dataset.hjson\")\n",
+    "allen_ephys_mapping = DictionaryMapping.load(\"https://raw.githubusercontent.com/BlueBrain/nexus/d9f6cb83a27149c29bc604f3c34ea00c9ad64e67/docs/src/main/paradox/docs/getting-started/notebooks/mappings/allen_ephys_dataset.hjson\")\n",
     "nephys_allen_resources = forge.map(nm_allen_metadata, allen_ephys_mapping, na='')"
    ]
   },

diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/ProjectMapper.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/ProjectMapper.scala
@@ -0,0 +1,19 @@
+package ch.epfl.bluebrain.nexus.ship
+
+import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef
+import ch.epfl.bluebrain.nexus.ship.config.ShipConfig.ProjectMapping
+
+trait ProjectMapper {
+  def map(project: ProjectRef): ProjectRef
+}
+
+object ProjectMapper {
+
+  def apply(projectMapping: ProjectMapping): ProjectMapper =
+    (project: ProjectRef) =>
+      projectMapping match {
+        case m if m.isEmpty => project
+        case mapping        => mapping.getOrElse(project, project)
+      }
+
+}
diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala
@@ -40,6 +40,7 @@ class RunShip {
                   val fetchContext   = FetchContext(ApiMappings.empty, xas, Quotas.disabled)
                   val eventLogConfig = config.eventLog
                   val baseUri        = config.baseUri
+                  val projectMapper  = ProjectMapper(config.projectMapping)
                   for {
                     // Provision organizations
                     _                           <- orgProvider.create(config.organizations.values)
@@ -50,24 +51,18 @@ class RunShip {
                     (schemaLog, fetchSchema)    <- SchemaWiring(config.eventLog, eventClock, xas, jsonLdApi)
                     (resourceLog, fetchResource) =
                       ResourceWiring(fetchContext, fetchSchema, eventLogConfig, eventClock, xas)
-                    rcr                         <- ContextWiring
-                                                     .resolverContextResolution(fetchResource, fetchContext, eventLogConfig, eventClock, xas)
-                    schemaImports                = SchemaWiring.schemaImports(
-                                                     fetchResource,
-                                                     fetchSchema,
-                                                     fetchContext,
-                                                     eventLogConfig,
-                                                     eventClock,
-                                                     xas
-                                                   )
+                    // format: off
+                    rcr                         <- ContextWiring.resolverContextResolution(fetchResource, fetchContext, eventLogConfig, eventClock, xas)
+                    schemaImports                = SchemaWiring.schemaImports(fetchResource, fetchSchema, fetchContext, eventLogConfig, eventClock, xas)
                     // Processors
-                    projectProcessor            <- ProjectProcessor(fetchActiveOrg, eventLogConfig, eventClock, xas)(baseUri)
-                    resolverProcessor            = ResolverProcessor(fetchContext, eventLogConfig, eventClock, xas)
-                    schemaProcessor              = SchemaProcessor(schemaLog, fetchContext, schemaImports, rcr, eventClock)
-                    resourceProcessor            = ResourceProcessor(resourceLog, fetchContext, eventClock)
-                    esViewsProcessor            <- ElasticSearchViewProcessor(fetchContext, rcr, eventLogConfig, eventClock, xas)
-                    bgViewsProcessor             = BlazegraphViewProcessor(fetchContext, rcr, eventLogConfig, eventClock, xas)
-                    compositeViewsProcessor      = CompositeViewProcessor(fetchContext, rcr, eventLogConfig, eventClock, xas)
+                    projectProcessor            <- ProjectProcessor(fetchActiveOrg, projectMapper, eventLogConfig, eventClock, xas)(baseUri)
+                    resolverProcessor            = ResolverProcessor(fetchContext, projectMapper, eventLogConfig, eventClock, xas)
+                    schemaProcessor              = SchemaProcessor(schemaLog, fetchContext, schemaImports, rcr, projectMapper, eventClock)
+                    resourceProcessor            = ResourceProcessor(resourceLog, projectMapper, fetchContext, eventClock)
+                    esViewsProcessor            <- ElasticSearchViewProcessor(fetchContext, rcr, projectMapper, eventLogConfig, eventClock, xas)
+                    bgViewsProcessor             = BlazegraphViewProcessor(fetchContext, rcr, projectMapper, eventLogConfig, eventClock, xas)
+                    compositeViewsProcessor      = CompositeViewProcessor(fetchContext, rcr, projectMapper, eventLogConfig, eventClock, xas)
+                    // format: on
                     report                      <- EventProcessor
                                                      .run(
                                                        events,

diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfig.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfig.scala
@@ -1,26 +1,40 @@
 package ch.epfl.bluebrain.nexus.ship.config
 
 import cats.effect.IO
+import cats.implicits._
 import ch.epfl.bluebrain.nexus.delta.kernel.config.Configs
 import ch.epfl.bluebrain.nexus.delta.sdk.model.{BaseUri, ServiceAccountConfig}
 import ch.epfl.bluebrain.nexus.delta.sourcing.config.{DatabaseConfig, EventLogConfig}
+import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef
+import ch.epfl.bluebrain.nexus.ship.config.ShipConfig.ProjectMapping
 import com.typesafe.config.Config
 import fs2.io.file.Path
 import pureconfig.ConfigReader
+import pureconfig.configurable.genericMapReader
+import pureconfig.error.CannotConvert
 import pureconfig.generic.semiauto.deriveReader
 
 final case class ShipConfig(
     baseUri: BaseUri,
     database: DatabaseConfig,
     eventLog: EventLogConfig,
     organizations: OrganizationCreationConfig,
+    projectMapping: ProjectMapping = Map.empty,
     serviceAccount: ServiceAccountConfig
 )
 
 object ShipConfig {
 
-  implicit final val shipConfigReader: ConfigReader[ShipConfig] =
+  type ProjectMapping = Map[ProjectRef, ProjectRef]
+
+  implicit val mapReader: ConfigReader[ProjectMapping] =
+    genericMapReader(str =>
+      ProjectRef.parse(str).leftMap(e => CannotConvert(str, classOf[ProjectRef].getSimpleName, e))
+    )
+
+  implicit final val shipConfigReader: ConfigReader[ShipConfig] = {
     deriveReader[ShipConfig]
+  }
 
   def merge(externalConfigPath: Option[Path]): IO[(ShipConfig, Config)] =
     for {

diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/projects/ProjectProcessor.scala
@@ -15,11 +15,15 @@ import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.{EntityType, ProjectRef}
 import ch.epfl.bluebrain.nexus.ship.error.ShipError.ProjectDeletionIsNotAllowed
 import ch.epfl.bluebrain.nexus.ship.projects.ProjectProcessor.logger
-import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, EventUUIDF, ImportStatus}
+import ch.epfl.bluebrain.nexus.ship.{EventClock, EventProcessor, EventUUIDF, ImportStatus, ProjectMapper}
 import io.circe.Decoder
 
-final class ProjectProcessor private (projects: Projects, clock: EventClock, uuidF: EventUUIDF)
-    extends EventProcessor[ProjectEvent] {
+final class ProjectProcessor private (
+    projects: Projects,
+    projectMapper: ProjectMapper,
+    clock: EventClock,
+    uuidF: EventUUIDF
+) extends EventProcessor[ProjectEvent] {
   override def resourceType: EntityType = Projects.entityType
 
   override def decoder: Decoder[ProjectEvent] = ProjectEvent.serializer.codec
@@ -34,8 +38,9 @@ final class ProjectProcessor private (projects: Projects, clock: EventClock, uui
 
   private def evaluateInternal(event: ProjectEvent): IO[ImportStatus] = {
     implicit val s: Subject = event.subject
-    val projectRef          = event.project
+    val projectRef          = projectMapper.map(event.project)
     val cRev                = event.rev - 1
+
     event match {
       case ProjectCreated(_, _, _, _, _, description, apiMappings, base, vocab, enforceSchema, _, _) =>
         val fields = ProjectFields(description, apiMappings, Some(base), Some(vocab), enforceSchema)
@@ -63,8 +68,14 @@ final class ProjectProcessor private (projects: Projects, clock: EventClock, uui
 object ProjectProcessor {
 
   private val logger      = Logger[ProjectProcessor]
-  def apply(fetchActiveOrg: FetchActiveOrganization, config: EventLogConfig, clock: EventClock, xas: Transactors)(
-      implicit base: BaseUri
+  def apply(
+      fetchActiveOrg: FetchActiveOrganization,
+      projectMapper: ProjectMapper,
+      config: EventLogConfig,
+      clock: EventClock,
+      xas: Transactors
+  )(implicit
+      base: BaseUri
   ): IO[ProjectProcessor] =
     for {
       uuidF <- EventUUIDF.init()
@@ -79,6 +90,6 @@ object ProjectProcessor {
         xas,
         clock
       )(base, uuidF)
-      new ProjectProcessor(projects, clock, uuidF)
+      new ProjectProcessor(projects, projectMapper, clock, uuidF)
     }
 }