Merge branch 'master' into 4755-improve-error-logging

ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/Main.scala

@@ -19,8 +19,10 @@ object Main
 
   private val logger = Logger[Main.type]
 
-  private val inputFile: Opts[Path] =
-    Opts.option[String]("file", help = "The data file containing the imports.").map(Path(_))
+  private val inputPath: Opts[Path] =
+    Opts
+      .option[String]("path", help = "The path containing the imports. Either a single file or a directory")
+      .map(Path(_))
 
   private val configFile: Opts[Option[Path]] =
     Opts.option[String]("config", help = "The configuration file.").map(Path(_)).orNone
@@ -31,7 +33,7 @@ object Main
     .withDefault(Offset.start)
 
   private val run = Opts.subcommand("run", "Run an import") {
-    (inputFile, configFile, offset).mapN(Run)
+    (inputPath, configFile, offset).mapN(Run)
   }
 
   private val showConfig = Opts.subcommand("config", "Show reconciled config") {
@@ -41,7 +43,7 @@ object Main
   override def main: Opts[IO[ExitCode]] =
     (run orElse showConfig)
       .map {
-        case Run(file, config, offset) => new RunShip().run(file, config, offset)
+        case Run(path, config, offset) => new RunShip().run(path, config, offset)
         case ShowConfig(config)        => showConfig(config)
       }
       .map(_.as(ExitCode.Success))
@@ -55,7 +57,7 @@ object Main
 
   sealed private trait Command
 
-  final private case class Run(file: Path, config: Option[Path], offset: Offset) extends Command
+  final private case class Run(path: Path, config: Option[Path], offset: Offset) extends Command
 
   final private case class ShowConfig(config: Option[Path]) extends Command
 

ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/RunShip.scala

@@ -27,13 +27,13 @@ class RunShip {
 
   private val logger = Logger[RunShip]
 
-  def run(file: Path, config: Option[Path], fromOffset: Offset = Offset.start): IO[ImportReport] = {
+  def run(path: Path, config: Option[Path], fromOffset: Offset = Offset.start): IO[ImportReport] = {
     val clock                         = Clock[IO]
     val uuidF                         = UUIDF.random
     // Resources may have been created with different configurations so we adopt the lenient one for the import
     implicit val jsonLdApi: JsonLdApi = JsonLdJavaApi.lenient
     for {
-      _      <- logger.info(s"Running the import with file $file, config $config")
+      _      <- logger.info(s"Running the import with file $path, config $config")
       config <- ShipConfig.load(config)
       report <- Transactors.init(config.database).use { xas =>
                   val orgProvider    =
@@ -45,7 +45,7 @@ class RunShip {
                   for {
                     // Provision organizations
                     _                           <- orgProvider.create(config.organizations.values)
-                    events                       = eventStream(file, fromOffset)
+                    events                       = eventStream(path, fromOffset)
                     fetchActiveOrg               = FetchActiveOrganization(xas)
                     // format: off
                     // Wiring
@@ -81,8 +81,9 @@ class RunShip {
     } yield report
   }
 
-  private def eventStream(file: Path, fromOffset: Offset): Stream[IO, RowEvent] =
-    Files[IO]
+  private def eventStream(path: Path, fromOffset: Offset): Stream[IO, RowEvent] = {
+
+    def streamFromFile(file: Path) = Files[IO]
       .readUtf8Lines(file)
       .zipWithIndex
       .evalMap { case (line, index) =>
@@ -94,4 +95,20 @@ class RunShip {
         event.ordering.value >= fromOffset.value
       }
 
+    def streamFromDirectory(dirPath: Path) = {
+      val importFiles = Files[IO]
+        .list(dirPath)
+        .filter(_.extName.contains("json"))
+        .compile
+        .toList
+        .map(_.sortBy(_.fileName.toString))
+      Stream.evals(importFiles).flatMap(streamFromFile)
+    }
+
+    Stream.eval(Files[IO].isDirectory(path)).flatMap { isDir =>
+      if (isDir) streamFromDirectory(path)
+      else streamFromFile(path)
+    }
+  }
+
 }

ship/src/test/resources/import/multi-part-import/2024-04-05T14:38:31.165389Z.json

@@ -0,0 +1,6 @@
+{"ordering":2163821,"type":"project","org":"public","project":"sscx","id":"projects/public/sscx","rev":1,"value":{"rev": 1, "base": "https://bbp.epfl.ch/neurosciencegraph/data/", "uuid": "c7d70522-4305-480a-b190-75d757ed9a49", "@type": "ProjectCreated", "label": "sscx", "vocab": "https://bbp.epfl.ch/nexus/v1/resources/public/sscx/_/", "instant": "2020-01-22T16:03:33.105Z", "subject": {"@type": "User", "realm": "bbp", "subject": "bob"}, "apiMappings": {"prov": "http://www.w3.org/ns/prov#", "context": "https://incf.github.io/neuroshapes/contexts/", "schemaorg": "http://schema.org/", "datashapes": "https://neuroshapes.org/dash/", "ontologies": "https://neuroshapes.org/dash/ontology", "taxonomies": "https://neuroshapes.org/dash/taxonomy", "commonshapes": "https://neuroshapes.org/commons/", "provdatashapes": "https://provshapes.org/datashapes/", "provcommonshapes": "https://provshapes.org/commons/"}, "description": "This project contains somatosensorycortex dissemination publication data.", "organizationUuid": "d098a020-508b-4131-a28d-75f73b7f5f0e", "organizationLabel": "public"},"instant":"2020-01-22T17:03:33.105+01:00"}
+{"ordering":2163822,"type":"resolver","org":"public","project":"sscx","id":"https://bluebrain.github.io/nexus/vocabulary/defaultInProject","rev":1,"value":{"id": "https://bluebrain.github.io/nexus/vocabulary/defaultInProject", "rev": 1, "@type": "ResolverCreated", "value": {"name": "Default resolver", "@type": "InProjectValue", "priority": 1, "description": "Resolver created with the project"}, "source": {"@id": "https://bluebrain.github.io/nexus/vocabulary/defaultInProject", "@type": ["InProject", "Resolver"], "@context": "https://bluebrain.github.io/nexus/contexts/resolvers.json", "priority": 1}, "instant": "2020-01-22T16:03:34.841Z", "project": "public/sscx", "subject": {"@type": "User", "realm": "bbp", "subject": "bob"}},"instant":"2020-01-22T17:03:34.841+01:00"}
+{"ordering":2163826,"type":"resolver","org":"public","project":"sscx","id":"https://bluebrain.github.io/nexus/vocabulary/crossProject1","rev":1,"value":{"id": "https://bluebrain.github.io/nexus/vocabulary/crossProject1", "rev": 1, "@type": "ResolverCreated", "value": {"@type": "CrossProjectValue", "priority": 40, "projects": ["neurosciencegraph/datamodels"], "resourceTypes": [], "identityResolution": {"@type": "ProvidedIdentities", "value": [{"@type": "Authenticated", "realm": "bbp"}]}}, "source": {"@id": "https://bluebrain.github.io/nexus/vocabulary/crossProject1", "@type": ["Resolver", "CrossProject"], "@context": "https://bluebrain.github.io/nexus/contexts/resolvers.json", "priority": 50, "projects": ["neurosciencegraph/datamodels"], "identities": [{"@id": "https://bbp.epfl.ch/nexus/v1/realms/bbp/authenticated", "@type": "Authenticated", "realm": "bbp"}]}, "instant": "2020-01-22T16:03:35.040Z", "project": "public/sscx", "subject": {"@type": "User", "realm": "bbp", "subject": "bob"}},"instant":"2020-01-22T17:03:35.04+01:00"}
+{"ordering":2173453,"type":"resolver","org":"public","project":"sscx","id":"https://bluebrain.github.io/nexus/vocabulary/crossProject1","rev":2,"value":{"id": "https://bluebrain.github.io/nexus/vocabulary/crossProject1", "rev": 2, "tpe": "CrossProject", "@type": "ResolverDeprecated", "instant": "2020-01-28T08:23:36.270Z", "project": "public/sscx", "subject": {"@type": "User", "realm": "bbp", "subject": "bob"}},"instant":"2020-01-28T09:23:36.27+01:00"}
+{"ordering":2408475,"type":"resolver","org":"public","project":"sscx","id":"https://bluebrain.github.io/nexus/vocabulary/crossProject2","rev":1,"value":{"id": "https://bluebrain.github.io/nexus/vocabulary/crossProject2", "rev": 1, "@type": "ResolverCreated", "value": {"@type": "CrossProjectValue", "priority": 50, "projects": ["neurosciencegraph/datamodels"], "resourceTypes": [], "identityResolution": {"@type": "ProvidedIdentities", "value": [{"@type": "Authenticated", "realm": "bbp"}]}}, "source": {"@id": "https://bluebrain.github.io/nexus/vocabulary/crossProject2", "@type": ["CrossProject", "Resolver"], "@context": "https://bluebrain.github.io/nexus/contexts/resolvers.json", "priority": 50, "projects": ["neurosciencegraph/datamodels"], "identities": [{"@id": "https://bbp.epfl.ch/nexus/v1/realms/bbp/authenticated", "@type": "Authenticated", "realm": "bbp"}]}, "instant": "2020-03-09T13:36:19.246Z", "project": "public/sscx", "subject": {"@type": "User", "realm": "bbp", "subject": "bob"}},"instant":"2020-03-09T14:36:19.246+01:00"}
+{"ordering":4879496,"type":"resolver","org":"public","project":"sscx","id":"https://bluebrain.github.io/nexus/vocabulary/crossProject2","rev":2,"value":{"id": "https://bluebrain.github.io/nexus/vocabulary/crossProject2", "rev": 2, "@type": "ResolverUpdated", "value": {"@type": "CrossProjectValue", "priority": 50, "projects": ["neurosciencegraph/datamodels"], "resourceTypes": [], "identityResolution": {"@type": "UseCurrentCaller"}}, "source": {"@id": "https://bluebrain.github.io/nexus/vocabulary/crossProject2", "@type": ["Resolver", "CrossProject"], "@context": ["https://bluebrain.github.io/nexus/contexts/resolvers.json", "https://bluebrain.github.io/nexus/contexts/metadata.json"], "priority": 50, "projects": ["neurosciencegraph/datamodels"], "resourceTypes": [], "useCurrentCaller": true}, "instant": "2022-11-16T13:42:07.498Z", "project": "public/sscx", "subject": {"@type": "User", "realm": "bbp", "subject": "bob"}},"instant":"2022-11-16T14:42:07.498+01:00"}

ship/src/test/resources/import/multi-part-import/2024-04-06T11:34:31.165389Z.json

@@ -0,0 +1,6 @@
+{"ordering":4900000,"type":"resource","org":"public","project":"sscx","id":"https://bluebrain.github.io/nexus/vocabulary/crossProject2","rev":1,"value":{"id":"https://staging.nise.bbp.epfl.ch/nexus/v1/resources/tests/oliver/_/6285253a-0ed4-4d98-8821-fe030c3fef40","rev":1,"@type":"ResourceCreated","types":[],"schema":"https://bluebrain.github.io/nexus/schemas/unconstrained.json?rev=1","source":{"hello":"world"},"instant":"2024-03-15T17:20:08.294513Z","project":"public/sscx","subject":{"@type":"User","realm":"bbp","subject":"grabinsk"},"expanded":[{"@id":"https://staging.nise.bbp.epfl.ch/nexus/v1/resources/tests/oliver/_/6285253a-0ed4-4d98-8821-fe030c3fef40","https://staging.nise.bbp.epfl.ch/nexus/v1/vocabs/tests/oliver/hello":[{"@value":"world"}]}],"compacted":{"@id":"6285253a-0ed4-4d98-8821-fe030c3fef40","hello":"world","@context":{"@base":"https://staging.nise.bbp.epfl.ch/nexus/v1/resources/tests/oliver/_/","@vocab":"https://staging.nise.bbp.epfl.ch/nexus/v1/vocabs/tests/oliver/"}},"schemaProject":"tests/oliver","remoteContexts":[]},"instant":"2099-12-30T23:59:59.999+01:00"}
+{"ordering":5300965,"type":"project","org":"public","project":"sscx","id":"projects/public/sscx","rev":2,"value":{"rev": 2, "base": "https://bbp.epfl.ch/neurosciencegraph/data/", "uuid": "c7d70522-4305-480a-b190-75d757ed9a49", "@type": "ProjectUpdated", "label": "sscx", "vocab": "https://bbp.epfl.ch/ontologies/core/bmo/", "instant": "2023-07-16T18:42:59.530Z", "subject": {"@type": "User", "realm": "bbp", "subject": "alice"}, "apiMappings": {"prov": "http://www.w3.org/ns/prov#", "context": "https://incf.github.io/neuroshapes/contexts/", "schemaorg": "http://schema.org/", "datashapes": "https://neuroshapes.org/dash/", "ontologies": "https://neuroshapes.org/dash/ontology", "taxonomies": "https://neuroshapes.org/dash/taxonomy", "commonshapes": "https://neuroshapes.org/commons/", "provdatashapes": "https://provshapes.org/datashapes/", "provcommonshapes": "https://provshapes.org/commons/"}, "description": "This project contains somatosensorycortex dissemination publication data.", "organizationUuid": "d098a020-508b-4131-a28d-75f73b7f5f0e", "organizationLabel": "public"},"instant":"2023-07-16T20:42:59.53+02:00"}
+{"ordering":5318566,"type":"project","org":"public","project":"sscx","id":"projects/public/sscx","rev":3,"value":{"rev": 3, "base": "https://bbp.epfl.ch/nexus/v1/resources/public/sscx/_/", "uuid": "c7d70522-4305-480a-b190-75d757ed9a49", "@type": "ProjectUpdated", "label": "sscx", "vocab": "https://bbp.epfl.ch/ontologies/core/bmo/", "instant": "2023-07-21T13:55:02.463Z", "subject": {"@type": "User", "realm": "bbp", "subject": "alice"}, "apiMappings": {"prov": "http://www.w3.org/ns/prov#", "context": "https://incf.github.io/neuroshapes/contexts/", "schemaorg": "http://schema.org/", "datashapes": "https://neuroshapes.org/dash/", "ontologies": "https://neuroshapes.org/dash/ontology", "taxonomies": "https://neuroshapes.org/dash/taxonomy", "commonshapes": "https://neuroshapes.org/commons/", "provdatashapes": "https://provshapes.org/datashapes/", "provcommonshapes": "https://provshapes.org/commons/"}, "description": "This project contains somatosensorycortex dissemination publication data.", "organizationUuid": "d098a020-508b-4131-a28d-75f73b7f5f0e", "organizationLabel": "public"},"instant":"2023-07-21T15:55:02.463+02:00"}
+{"ordering":5418473,"type":"project","org":"public","project":"sscx","id":"projects/public/sscx","rev":4,"value":{"rev": 4, "base": "https://bbp.epfl.ch/data/public/sscx/", "uuid": "c7d70522-4305-480a-b190-75d757ed9a49", "@type": "ProjectUpdated", "label": "sscx", "vocab": "https://bbp.epfl.ch/ontologies/core/bmo/", "instant": "2023-08-22T15:05:13.654Z", "subject": {"@type": "User", "realm": "bbp", "subject": "alice"}, "apiMappings": {"prov": "http://www.w3.org/ns/prov#", "context": "https://incf.github.io/neuroshapes/contexts/", "schemaorg": "http://schema.org/", "datashapes": "https://neuroshapes.org/dash/", "ontologies": "https://neuroshapes.org/dash/ontology", "taxonomies": "https://neuroshapes.org/dash/taxonomy", "commonshapes": "https://neuroshapes.org/commons/", "provdatashapes": "https://provshapes.org/datashapes/", "provcommonshapes": "https://provshapes.org/commons/"}, "description": "This project contains somatosensorycortex dissemination publication data.", "organizationUuid": "d098a020-508b-4131-a28d-75f73b7f5f0e", "organizationLabel": "public"},"instant":"2023-08-22T17:05:13.654+02:00"}
+{"ordering":9999998,"type":"project","org":"public","project":"sscx","id":"projects/public/sscx","rev":5,"value":{"rev": 5, "uuid": "c7d70522-4305-480a-b190-75d757ed9a49", "@type": "ProjectDeprecated", "label": "sscx", "instant": "2099-12-30T23:59:59.999+01:00", "subject": {"@type": "User", "realm": "bbp", "subject": "bob"}, "organizationUuid": "d098a020-508b-4131-a28d-75f73b7f5f0e", "organizationLabel": "public"},"instant":"2099-12-30T23:59:59.999+01:00"}
+{"ordering":9999999,"type":"xxx",    "org":"public","project":"sscx","id":"https://bluebrain.github.io/nexus/vocabulary/xxx","rev":1,"value":{},"instant":"2099-12-31T23:59:59.999+01:00"}

ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/RunShipSuite.scala

@@ -33,7 +33,7 @@ class RunShipSuite extends NexusSuite with RunShipSuite.Fixture {
     ()
   }
 
-  test("Run import") {
+  test("Run import by providing the path to a file") {
     val expected = ImportReport(
       Offset.at(9999999L),
       Instant.parse("2099-12-31T22:59:59.999Z"),
@@ -50,6 +50,23 @@ class RunShipSuite extends NexusSuite with RunShipSuite.Fixture {
     } yield ()
   }
 
+  test("Run import by providing the path to a directory") {
+    val expected = ImportReport(
+      Offset.at(9999999L),
+      Instant.parse("2099-12-31T22:59:59.999Z"),
+      Map(
+        Projects.entityType  -> Count(5L, 0L),
+        Resolvers.entityType -> Count(5L, 0L),
+        Resources.entityType -> Count(1L, 0L),
+        EntityType("xxx")    -> Count(0L, 1L)
+      )
+    )
+    for {
+      importFile <- asPath("import/multi-part-import")
+      _          <- new RunShip().run(importFile, None).assertEquals(expected)
+    } yield ()
+  }
+
   test("Test the increment") {
     for {
       importFileWithTwoProjects <- asPath("import/two-projects.json")

tests/docker/docker-compose.yml

@@ -198,7 +198,7 @@ services:
 #    depends_on:
 #      - postgres
 #    image: bluebrain/nexus-ship:latest
-#    command: run --file "/tmp/export.json" --config "/config/external-ship.conf"
+#    command: run --path "/tmp/export.json" --config "/config/external-ship.conf"
 #    volumes:
 #      - /tmp/ship:/tmp # put the export file in /tmp/ship locally
 #      - ./config:/config