Implement custom media type detection (#4312)

* Implement custom content type detection --------- Co-authored-by: Simon Dumas <[email protected]>
BlueBrain · Oct 4, 2023 · cefab5d · cefab5d
1 parent 523ff52
commit cefab5d
Show file tree

Hide file tree

Showing 53 changed files with 994 additions and 509 deletions.
diff --git a/build.sbt b/build.sbt
@@ -21,7 +21,6 @@ val akkaCorsVersion         = "1.2.0"
 val akkaVersion             = "2.6.21"
 val alpakkaVersion          = "3.0.4"
 val apacheCompressVersion   = "1.24.0"
-val apacheIoVersion         = "1.3.2"
 val awsSdkVersion           = "2.17.184"
 val byteBuddyAgentVersion   = "1.10.17"
 val betterMonadicForVersion = "0.3.1"
@@ -77,7 +76,6 @@ lazy val alpakkaFile        = "com.lightbend.akka"           %% "akka-stream-alp
 lazy val alpakkaSse         = "com.lightbend.akka"           %% "akka-stream-alpakka-sse"  % alpakkaVersion
 lazy val alpakkaS3          = "com.lightbend.akka"           %% "akka-stream-alpakka-s3"   % alpakkaVersion
 lazy val apacheCompress     = "org.apache.commons"            % "commons-compress"         % apacheCompressVersion
-lazy val apacheIo           = "org.apache.commons"            % "commons-io"               % apacheIoVersion
 lazy val awsSdk             = "software.amazon.awssdk"        % "s3"                       % awsSdkVersion
 lazy val betterMonadicFor   = "com.olegpy"                   %% "better-monadic-for"       % betterMonadicForVersion
 lazy val byteBuddyAgent     = "net.bytebuddy"                 % "byte-buddy-agent"         % byteBuddyAgentVersion
@@ -206,6 +204,8 @@ lazy val kernel = project
   .settings(shared, compilation, coverage, release, assertJavaVersion)
   .settings(
     libraryDependencies  ++= Seq(
+      akkaActorTyped, // Needed to create content type
+      akkaHttpCore,
       caffeine,
       catsRetry,
       circeCore,
@@ -216,6 +216,7 @@ lazy val kernel = project
       log4cats,
       pureconfig,
       scalaLogging,
+      munit     % Test,
       scalaTest % Test
     ),
     addCompilerPlugin(kindProjector),
@@ -734,6 +735,7 @@ lazy val storage = project
     servicePackaging,
     coverageMinimumStmtTotal := 75
   )
+  .dependsOn(kernel)
   .settings(cargo := {
     import scala.sys.process._
 
@@ -753,10 +755,8 @@ lazy val storage = project
     buildInfoKeys            := Seq[BuildInfoKey](version),
     buildInfoPackage         := "ch.epfl.bluebrain.nexus.storage.config",
     Docker / packageName     := "nexus-storage",
-    javaSpecificationVersion := "1.8",
     libraryDependencies     ++= Seq(
       apacheCompress,
-      apacheIo,
       akkaHttp,
       akkaHttpCirce,
       akkaStream,
@@ -773,6 +773,7 @@ lazy val storage = project
       akkaHttpTestKit % Test,
       akkaTestKit     % Test,
       mockito         % Test,
+      munit           % Test,
       scalaTest       % Test
     ),
     cleanFiles              ++= Seq(

diff --git a/...el/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/http/MediaTypeDetectorConfig.scala b/...el/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/http/MediaTypeDetectorConfig.scala
@@ -0,0 +1,41 @@
+package ch.epfl.bluebrain.nexus.delta.kernel.http
+
+import akka.http.scaladsl.model.MediaType
+import cats.syntax.all._
+import pureconfig.ConfigReader
+import pureconfig.configurable.genericMapReader
+import pureconfig.error.CannotConvert
+
+/**
+  * Allows to define custom media types for the given extensions
+  */
+final case class MediaTypeDetectorConfig(extensions: Map[String, MediaType]) {
+  def find(extension: String): Option[MediaType] = extensions.get(extension)
+
+}
+
+object MediaTypeDetectorConfig {
+
+  val Empty = new MediaTypeDetectorConfig(Map.empty)
+
+  def apply(values: (String, MediaType)*) = new MediaTypeDetectorConfig(values.toMap)
+
+  implicit final val mediaTypeDetectorConfigReader: ConfigReader[MediaTypeDetectorConfig] = {
+    implicit val mediaTypeConfigReader: ConfigReader[MediaType]  =
+      ConfigReader.fromString(str =>
+        MediaType
+          .parse(str)
+          .leftMap(_ => CannotConvert(str, classOf[MediaType].getSimpleName, s"'$str' is not a valid content type."))
+      )
+    implicit val mapReader: ConfigReader[Map[String, MediaType]] = genericMapReader(Right(_))
+
+    ConfigReader.fromCursor { cursor =>
+      for {
+        obj           <- cursor.asObjectCursor
+        extensionsKey <- obj.atKey("extensions")
+        extensions    <- ConfigReader[Map[String, MediaType]].from(extensionsKey)
+      } yield MediaTypeDetectorConfig(extensions)
+    }
+  }
+
+}
diff --git a/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/FileUtils.scala b/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/FileUtils.scala
@@ -0,0 +1,15 @@
+package ch.epfl.bluebrain.nexus.delta.kernel.utils
+
+object FileUtils {
+
+  /**
+    * Extracts the extension from the given filename
+    */
+  def extension(filename: String): Option[String] = {
+    val lastDotIndex = filename.lastIndexOf('.')
+    Option.when(lastDotIndex >= 0) {
+      filename.substring(lastDotIndex + 1)
+    }
+  }
+
+}
diff --git a/...c/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/http/MediaTypeDetectorConfigSuite.scala b/...c/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/http/MediaTypeDetectorConfigSuite.scala
@@ -0,0 +1,54 @@
+package ch.epfl.bluebrain.nexus.delta.kernel.http
+
+import akka.http.scaladsl.model.ContentTypes
+import munit.FunSuite
+import pureconfig.ConfigSource
+
+class MediaTypeDetectorConfigSuite extends FunSuite {
+
+  private def parseConfig(value: String) =
+    ConfigSource.string(value).at("media-type-detector").load[MediaTypeDetectorConfig]
+
+  test("Parse successfully the config with no defined extension") {
+    val config = parseConfig(
+      """
+        |media-type-detector {
+        | extensions {
+        | }
+        |}
+        |""".stripMargin
+    )
+
+    val expected = MediaTypeDetectorConfig.Empty
+    assertEquals(config, Right(expected))
+  }
+
+  test("Parse successfully the config") {
+    val config = parseConfig(
+      """
+        |media-type-detector {
+        | extensions {
+        |   json = application/json
+        | }
+        |}
+        |""".stripMargin
+    )
+
+    val expected = MediaTypeDetectorConfig("json" -> ContentTypes.`application/json`.mediaType)
+    assertEquals(config, Right(expected))
+  }
+
+  test("Fail to parse the config with an invalid content type") {
+    val config = parseConfig(
+      """
+        |media-type-detector {
+        | extensions {
+        |   json = xxx
+        | }
+        |}
+        |""".stripMargin
+    )
+
+    assert(config.isLeft, "Parsing must fail with an invalid content type")
+  }
+}
diff --git a/delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/FileUtilsSuite.scala b/delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/FileUtilsSuite.scala
@@ -0,0 +1,25 @@
+package ch.epfl.bluebrain.nexus.delta.kernel.utils
+
+import munit.FunSuite
+
+class FileUtilsSuite extends FunSuite {
+
+  test("Detect json extension") {
+    val obtained = FileUtils.extension("my-file.json")
+    val expected = Some("json")
+    assertEquals(obtained, expected)
+  }
+
+  test("Detect zip extension") {
+    val obtained = FileUtils.extension("my-file.json.zip")
+    val expected = Some("zip")
+    assertEquals(obtained, expected)
+  }
+
+  test("Detect no extension") {
+    val obtained = FileUtils.extension("my-file")
+    val expected = None
+    assertEquals(obtained, expected)
+  }
+
+}
diff --git a/delta/plugins/storage/src/main/resources/storage.conf b/delta/plugins/storage/src/main/resources/storage.conf
@@ -76,6 +76,13 @@ plugins.storage {
   files {
     # the files event log configuration
     event-log = ${app.defaults.event-log}
+
+    # Allows to define default media types for the given file extensions
+    media-type-detector {
+      extensions {
+        #extension = "application/custom"
+      }
+    }
   }
   defaults {
     # the name of the default storage

diff --git a/...ns/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala b/...ns/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala
@@ -720,7 +720,7 @@ object Files {
   ): Files = {
     implicit val classicAs: ClassicActorSystem = as.classicSystem
     new Files(
-      FormDataExtractor.apply,
+      FormDataExtractor(config.mediaTypeDetector),
       ScopedEventLog(definition, config.eventLog, xas),
       aclCheck,
       fetchContext,

diff --git a/...rage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesConfig.scala b/...rage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesConfig.scala
@@ -1,5 +1,6 @@
 package ch.epfl.bluebrain.nexus.delta.plugins.storage.files
 
+import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig
 import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig
 import pureconfig.ConfigReader
 import pureconfig.generic.semiauto.deriveReader
@@ -10,7 +11,7 @@ import pureconfig.generic.semiauto.deriveReader
   * @param eventLog
   *   configuration of the event log
   */
-final case class FilesConfig(eventLog: EventLogConfig)
+final case class FilesConfig(eventLog: EventLogConfig, mediaTypeDetector: MediaTypeDetectorConfig)
 
 object FilesConfig {
   implicit final val filesConfigReader: ConfigReader[FilesConfig] =