Init org provider + project processor (#4742)

* Init org provider + project processor

---------

Co-authored-by: Simon Dumas <[email protected]>

.github/workflows/ci-delta-plugins.yml

@@ -2,8 +2,8 @@ name: Delta Plugins unit tests
 on:
   pull_request:
     paths:
-      - 'ship/**'
       - 'delta/kernel/**'
+      - 'delta/plugins/**'
       - 'delta/rdf/**'
       - 'delta/sdk/**'
       - 'delta/sourcing-psql/**'

.github/workflows/ci-delta-ship.yml

@@ -2,8 +2,8 @@ name: Delta Ship unit tests
 on:
   pull_request:
     paths:
+      - 'ship/**'
       - 'delta/kernel/**'
-      - 'delta/plugins/**'
       - 'delta/rdf/**'
       - 'delta/sdk/**'
       - 'delta/sourcing-psql/**'

build.sbt

@@ -845,7 +845,7 @@ lazy val root = project
   .in(file("."))
   .settings(name := "nexus", moduleName := "nexus")
   .settings(compilation, shared, noPublish)
-  .aggregate(docs, delta, storage, tests)
+  .aggregate(docs, delta, ship, storage, tests)
 
 lazy val noPublish = Seq(
   publish / skip                         := true,

delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/config/AppConfig.scala

@@ -3,6 +3,7 @@ package ch.epfl.bluebrain.nexus.delta.config
 import cats.effect.IO
 import cats.syntax.all._
 import ch.epfl.bluebrain.nexus.delta.kernel.cache.CacheConfig
+import ch.epfl.bluebrain.nexus.delta.kernel.config.Configs
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApiConfig
 import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclsConfig
 import ch.epfl.bluebrain.nexus.delta.sdk.fusion.FusionConfig
@@ -20,9 +21,9 @@ import ch.epfl.bluebrain.nexus.delta.sdk.sse.SseConfig
 import ch.epfl.bluebrain.nexus.delta.sdk.typehierarchy.TypeHierarchyConfig
 import ch.epfl.bluebrain.nexus.delta.sourcing.config.{DatabaseConfig, ProjectionConfig}
 import ch.epfl.bluebrain.nexus.delta.sourcing.exporter.ExportConfig
-import com.typesafe.config.{Config, ConfigFactory, ConfigParseOptions, ConfigResolveOptions}
+import com.typesafe.config.Config
+import pureconfig.ConfigReader
 import pureconfig.generic.semiauto.deriveReader
-import pureconfig.{ConfigReader, ConfigSource}
 
 import java.io.{File, InputStreamReader}
 import java.nio.charset.StandardCharsets.UTF_8
@@ -56,9 +57,6 @@ final case class AppConfig(
 
 object AppConfig {
 
-  private val parseOptions    = ConfigParseOptions.defaults().setAllowMissing(false)
-  private val resolverOptions = ConfigResolveOptions.defaults()
-
   /**
     * Loads the application in two steps, wrapping the error type:
     *
@@ -85,30 +83,13 @@ object AppConfig {
       pluginsConfigPaths: List[String] = List.empty,
       accClassLoader: ClassLoader = getClass.getClassLoader
   ): IO[(AppConfig, Config)] = {
-
-    // Merge configs according to their order
-    def merge(configs: Config*) = IO.fromEither {
-      val merged = configs
-        .foldLeft(ConfigFactory.defaultOverrides())(_ withFallback _)
-        .withFallback(ConfigFactory.load())
-        .resolve(resolverOptions)
-      ConfigSource.fromConfig(merged).at("app").load[AppConfig].map(_ -> merged).leftMap(AppConfigError(_))
-    }
-
     for {
-      externalConfig            <- IO.blocking(externalConfigPath.fold(ConfigFactory.empty()) { p =>
-                                     ConfigFactory.parseFile(new File(p), parseOptions)
-                                   })
-      defaultConfig             <- IO.blocking(ConfigFactory.parseResources("default.conf", parseOptions))
-      pluginConfigs             <- IO.blocking {
-                                     pluginsConfigPaths.map { string =>
-                                       ConfigFactory.parseReader(
-                                         new InputStreamReader(accClassLoader.getResourceAsStream(string), UTF_8),
-                                         parseOptions
-                                       )
-                                     }
+      externalConfig            <- Configs.parseFile(externalConfigPath.map(new File(_)))
+      defaultConfig             <- Configs.parseResource("default.conf")
+      pluginConfigs             <- pluginsConfigPaths.traverse { path =>
+                                     Configs.parseReader(new InputStreamReader(accClassLoader.getResourceAsStream(path), UTF_8))
                                    }
-      (appConfig, mergedConfig) <- merge(externalConfig :: defaultConfig :: pluginConfigs: _*)
+      (appConfig, mergedConfig) <- Configs.merge[AppConfig]("app", externalConfig :: defaultConfig :: pluginConfigs: _*)
     } yield (appConfig, mergedConfig)
   }
 

delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/OrganizationsModule.scala

@@ -34,7 +34,7 @@ object OrganizationsModule extends ModuleDef {
     ) =>
       OrganizationsImpl(
         scopeInitializer,
-        config.organizations,
+        config.organizations.eventLog,
         xas,
         clock
       )(uuidF)

delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/ProjectsModule.scala

@@ -67,7 +67,7 @@ object ProjectsModule extends ModuleDef {
           ValidateProjectDeletion(xas, config.projects.deletion.enabled),
           scopeInitializer,
           mappings.merge,
-          config.projects,
+          config.projects.eventLog,
           xas,
           clock
         )(baseUri, uuidF)

delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/OrganizationsRoutesSpec.scala

@@ -10,7 +10,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.acls.model.AclAddress
 import ch.epfl.bluebrain.nexus.delta.sdk.generators.OrganizationGen
 import ch.epfl.bluebrain.nexus.delta.sdk.implicits._
 import ch.epfl.bluebrain.nexus.delta.sdk.organizations.model.OrganizationRejection.OrganizationNonEmpty
-import ch.epfl.bluebrain.nexus.delta.sdk.organizations.{OrganizationDeleter, OrganizationsConfig, OrganizationsImpl}
+import ch.epfl.bluebrain.nexus.delta.sdk.organizations.{OrganizationDeleter, OrganizationsImpl}
 import ch.epfl.bluebrain.nexus.delta.sdk.permissions.Permissions.{orgs => orgsPermissions}
 import ch.epfl.bluebrain.nexus.delta.sdk.utils.BaseRouteSpec
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.{Anonymous, Subject, User}
@@ -29,9 +29,7 @@ class OrganizationsRoutesSpec extends BaseRouteSpec {
   private val org1 = OrganizationGen.organization("org1", fixedUuid, Some("My description"))
   private val org2 = OrganizationGen.organization("org2", fixedUuid)
 
-  private val config = OrganizationsConfig(eventLogConfig, pagination)
-
-  private lazy val orgs                            = OrganizationsImpl(ScopeInitializer.noop, config, xas, clock)
+  private lazy val orgs                            = OrganizationsImpl(ScopeInitializer.noop, eventLogConfig, xas, clock)
   private lazy val orgDeleter: OrganizationDeleter = id => IO.raiseWhen(id == org1.label)(OrganizationNonEmpty(id))
 
   private val superUser                      = User("superUser", Label.unsafe(genString()))

delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/ProjectsRoutesSpec.scala

@@ -105,7 +105,7 @@ class ProjectsRoutesSpec extends BaseRouteSpec with BeforeAndAfterAll {
   )
 
   private lazy val projects     =
-    ProjectsImpl(fetchOrg, _ => IO.unit, ScopeInitializer.noop, defaultApiMappings, projectsConfig, xas, clock)
+    ProjectsImpl(fetchOrg, _ => IO.unit, ScopeInitializer.noop, defaultApiMappings, eventLogConfig, xas, clock)
   private lazy val provisioning =
     ProjectProvisioning(aclCheck.append, projects, provisioningConfig)
   private lazy val routes       = Route.seal(

delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/config/Configs.scala

@@ -0,0 +1,48 @@
+package ch.epfl.bluebrain.nexus.delta.kernel.config
+
+import cats.effect.IO
+import com.typesafe.config.{Config, ConfigFactory, ConfigParseOptions, ConfigResolveOptions}
+import cats.syntax.all._
+import pureconfig.error.ConfigReaderException
+import pureconfig.{ConfigReader, ConfigSource}
+
+import java.io.{File, Reader}
+
+object Configs {
+
+  private val parseOptions = ConfigParseOptions.defaults().setAllowMissing(false)
+
+  private val resolverOptions = ConfigResolveOptions.defaults()
+
+  /**
+    * Loads the config from the file or return an empty configuration
+    */
+  def parseFile(file: Option[File]): IO[Config] =
+    IO.blocking(file.fold(ConfigFactory.empty()) { f =>
+      ConfigFactory.parseFile(f, parseOptions)
+    })
+
+  /**
+    * Loads the config from resource
+    */
+  def parseResource(resource: String): IO[Config] =
+    IO.blocking(ConfigFactory.parseResources(resource, parseOptions))
+
+  /**
+    * Loads the config from the reader
+    */
+  def parseReader(reader: Reader): IO[Config] =
+    IO.blocking(ConfigFactory.parseReader(reader, parseOptions))
+
+  /**
+    * Merge the configs in order and load the namespace according to the config reader
+    */
+  def merge[C: ConfigReader](namespace: String, configs: Config*): IO[(C, Config)] = IO.fromEither {
+    val merged = configs
+      .foldLeft(ConfigFactory.defaultOverrides())(_ withFallback _)
+      .withFallback(ConfigFactory.load())
+      .resolve(resolverOptions)
+    ConfigSource.fromConfig(merged).at(namespace).load[C].map(_ -> merged).leftMap(ConfigReaderException(_))
+  }
+
+}

delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/auth/Credentials.scala

@@ -3,11 +3,8 @@ package ch.epfl.bluebrain.nexus.delta.sdk.auth
 import ch.epfl.bluebrain.nexus.delta.kernel.Secret
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label
 import pureconfig.ConfigReader
-import pureconfig.error.CannotConvert
 import pureconfig.generic.semiauto.deriveReader
 
-import scala.annotation.nowarn
-
 /**
   * Enumerates the different ways to obtain an auth toke for making requests to a remote service
   */
@@ -37,11 +34,7 @@ object Credentials {
     */
   case class ClientCredentials(user: String, password: Secret[String], realm: Label) extends Credentials
   object ClientCredentials {
-    @nowarn("cat=unused")
-    implicit private val labelConfigReader: ConfigReader[Label] = ConfigReader.fromString(str =>
-      Label(str).left.map(e => CannotConvert(str, classOf[Label].getSimpleName, e.getMessage))
-    )
-    implicit val configReader: ConfigReader[ClientCredentials]  = deriveReader[ClientCredentials]
+    implicit val configReader: ConfigReader[ClientCredentials] = deriveReader[ClientCredentials]
   }
 
   implicit val configReader: ConfigReader[Credentials] = deriveReader[Credentials]

delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/organizations/OrganizationsImpl.scala

@@ -13,6 +13,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.organizations.model.{OrganizationComman
 import ch.epfl.bluebrain.nexus.delta.sdk.syntax._
 import ch.epfl.bluebrain.nexus.delta.sdk.{OrganizationResource, ScopeInitializer}
 import ch.epfl.bluebrain.nexus.delta.sourcing._
+import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label
 
@@ -85,14 +86,14 @@ object OrganizationsImpl {
 
   def apply(
       scopeInitializer: ScopeInitializer,
-      config: OrganizationsConfig,
+      config: EventLogConfig,
       xas: Transactors,
       clock: Clock[IO]
   )(implicit
       uuidf: UUIDF
   ): Organizations =
     new OrganizationsImpl(
-      GlobalEventLog(Organizations.definition(clock), config.eventLog, xas),
+      GlobalEventLog(Organizations.definition(clock), config, xas),
       scopeInitializer
     )
 

delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/projects/ProjectsImpl.scala

@@ -15,6 +15,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ProjectRejection._
 import ch.epfl.bluebrain.nexus.delta.sdk.projects.model._
 import ch.epfl.bluebrain.nexus.delta.sdk.syntax._
 import ch.epfl.bluebrain.nexus.delta.sourcing._
+import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ElemStream, ProjectRef}
 import ch.epfl.bluebrain.nexus.delta.sourcing.offset.Offset
@@ -112,15 +113,15 @@ object ProjectsImpl {
       validateDeletion: ValidateProjectDeletion,
       scopeInitializer: ScopeInitializer,
       defaultApiMappings: ApiMappings,
-      config: ProjectsConfig,
+      config: EventLogConfig,
       xas: Transactors,
       clock: Clock[IO]
   )(implicit
       base: BaseUri,
       uuidF: UUIDF
   ): Projects =
     new ProjectsImpl(
-      ScopedEventLog(Projects.definition(fetchAndValidateOrg, validateDeletion, clock), config.eventLog, xas),
+      ScopedEventLog(Projects.definition(fetchAndValidateOrg, validateDeletion, clock), config, xas),
       scopeInitializer,
       defaultApiMappings
     )

delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/provisioning/AutomaticProvisioningConfig.scala

@@ -46,10 +46,6 @@ object AutomaticProvisioningConfig {
   implicit private val iriConfigReader: ConfigReader[Iri] =
     ConfigReader.fromString(str => Iri(str).leftMap(err => CannotConvert(str, classOf[Iri].getSimpleName, err)))
 
-  implicit private val labelConfigReader: ConfigReader[Label] = ConfigReader.fromString(str =>
-    Label(str).leftMap(e => CannotConvert(str, classOf[Label].getSimpleName, e.getMessage))
-  )
-
   implicit private val mapReader: ConfigReader[Map[Label, Label]] =
     genericMapReader(str => Label(str).leftMap(e => CannotConvert(str, classOf[Label].getSimpleName, e.getMessage)))
 

delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/organizations/OrganizationDeleterSuite.scala

@@ -35,7 +35,6 @@ class OrganizationDeleterSuite extends NexusSuite with ConfigFixtures with Proje
   }
 
   private val config              = ProjectsConfig(eventLogConfig, pagination, deletionConfig)
-  private val orgConfig           = OrganizationsConfig(eventLogConfig, pagination)
   private lazy val projectFixture = createProjectsFixture(fetchOrg, defaultApiMappings, config, clock)
 
   override def munitFixtures: Seq[AnyFixture[_]] = List(projectFixture)
@@ -44,7 +43,7 @@ class OrganizationDeleterSuite extends NexusSuite with ConfigFixtures with Proje
   private lazy val orgDeleter      = OrganizationDeleter(xas)
   private val projRef              = ProjectRef.unsafe(org1.value, "myproj")
   private val fields               = ProjectFields(None, ApiMappings.empty, None, None)
-  private lazy val orgs            = OrganizationsImpl(ScopeInitializer.noop, orgConfig, xas, clock)
+  private lazy val orgs            = OrganizationsImpl(ScopeInitializer.noop, eventLogConfig, xas, clock)
   private val permission           = Permissions.resources.read
   private lazy val acls            = AclsImpl(IO.pure(Set(permission)), _ => IO.unit, Set(), aclsConfig, xas, clock)
 

delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/organizations/OrganizationsImplSpec.scala

@@ -26,8 +26,6 @@ class OrganizationsImplSpec
     with CancelAfterFailure
     with ConfigFixtures {
 
-  private val config = OrganizationsConfig(eventLogConfig, pagination)
-
   val uuid                  = UUID.randomUUID()
   implicit val uuidF: UUIDF = UUIDF.fixed(uuid)
 
@@ -48,7 +46,7 @@ class OrganizationsImplSpec
       IO.unit
   }
 
-  private lazy val orgs = OrganizationsImpl(ScopeInitializer.noop, config, xas, clock)
+  private lazy val orgs = OrganizationsImpl(ScopeInitializer.noop, eventLogConfig, xas, clock)
 
   "Organizations implementation" should {
 
@@ -109,7 +107,7 @@ class OrganizationsImplSpec
 
     "run the initializer upon organization creation" in {
       val initializerWasExecuted = Ref.unsafe[IO, Boolean](false)
-      val orgs                   = OrganizationsImpl(orgInitializer(initializerWasExecuted), config, xas, clock)
+      val orgs                   = OrganizationsImpl(orgInitializer(initializerWasExecuted), eventLogConfig, xas, clock)
 
       orgs.create(Label.unsafe(genString()), description).accepted
       initializerWasExecuted.get.accepted shouldEqual true

delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/projects/ProjectsFixture.scala

@@ -41,7 +41,7 @@ trait ProjectsFixture { self: CatsEffectSuite =>
     ResourceSuiteLocalFixture(
       "projects",
       Doobie.resource().map { xas =>
-        (xas, ProjectsImpl(fetchOrgs, _ => IO.unit, inits, apiMappings, config, xas, clock))
+        (xas, ProjectsImpl(fetchOrgs, _ => IO.unit, inits, apiMappings, config.eventLog, xas, clock))
       }
     )
   }

delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/projects/ProjectsImplSpec.scala

@@ -53,8 +53,6 @@ class ProjectsImplSpec extends CatsEffectSpec with DoobieScalaTestFixture with C
 
   private val order = ResourceF.sortBy[Project]("_label").value
 
-  private val config = ProjectsConfig(eventLogConfig, pagination, deletionConfig)
-
   private def fetchOrg: FetchOrganization = {
     case `org1`          => IO.pure(Organization(org1, orgUuid, None))
     case `org2`          => IO.pure(Organization(org2, orgUuid, None))
@@ -82,7 +80,7 @@ class ProjectsImplSpec extends CatsEffectSpec with DoobieScalaTestFixture with C
   }
 
   private lazy val projects =
-    ProjectsImpl(fetchOrg, validateDeletion, ScopeInitializer.noop, defaultApiMappings, config, xas, clock)
+    ProjectsImpl(fetchOrg, validateDeletion, ScopeInitializer.noop, defaultApiMappings, eventLogConfig, xas, clock)
 
   "The Projects operations bundle" should {
     "create a project" in {
@@ -274,7 +272,7 @@ class ProjectsImplSpec extends CatsEffectSpec with DoobieScalaTestFixture with C
       val projectRef             = ProjectRef.unsafe("org", genString())
       val initializerWasExecuted = Ref.unsafe[IO, Boolean](false)
       // format: off
-      val projects = ProjectsImpl(fetchOrg, validateDeletion, projectInitializer(initializerWasExecuted), defaultApiMappings, config, xas, clock)
+      val projects = ProjectsImpl(fetchOrg, validateDeletion, projectInitializer(initializerWasExecuted), defaultApiMappings, eventLogConfig, xas, clock)
       // format: on
 
       projects.create(projectRef, payload)(Identity.Anonymous).accepted