Custom binary files are not compressed when fetched by the API (#4751)

* Custom binary files are not compressed when fetched by the API

---------

Co-authored-by: Simon Dumas <[email protected]>

delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/instances/ContentTypeInstances.scala

@@ -0,0 +1,39 @@
+package ch.epfl.bluebrain.nexus.delta.kernel.instances
+
+import akka.http.scaladsl.model.MediaType.NotCompressible
+import akka.http.scaladsl.model.{ContentType, MediaType, MediaTypes}
+import cats.syntax.all._
+import io.circe.{Decoder, Encoder}
+
+import java.util.Locale
+
+trait ContentTypeInstances {
+  implicit val contentTypeEncoder: Encoder[ContentType] =
+    Encoder.encodeString.contramap(_.value)
+
+  implicit val contentTypeDecoder: Decoder[ContentType] =
+    Decoder.decodeString.emap(
+      ContentType
+        .parse(_)
+        .bimap(
+          _.mkString("\n"),
+          markBinaryAsNonCompressible
+        )
+    )
+
+  /**
+    * When parsing a custom binary media type, akka assumes that it is compressible which is traduced by a performance
+    * hit when we compress responses so we revert this
+    */
+  private def markBinaryAsNonCompressible(contentType: ContentType) =
+    contentType match {
+      case b: ContentType.Binary if isCustomMediaType(b.mediaType) =>
+        ContentType.Binary(b.mediaType.withComp(NotCompressible))
+      case other                                                   => other
+    }
+
+  private def isCustomMediaType(mediaType: MediaType) =
+    MediaTypes
+      .getForKey(mediaType.mainType.toLowerCase(Locale.ROOT) -> mediaType.subType.toLowerCase(Locale.ROOT))
+      .isEmpty
+}

delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/instances/package.scala → delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/instances/instances.scala

@@ -1,3 +1,3 @@
-package ch.epfl.bluebrain.nexus.delta.plugins.storage
+package ch.epfl.bluebrain.nexus.delta.kernel
 
 package object instances extends ContentTypeInstances

delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/instances/ContentTypeInstancesSuite.scala

@@ -0,0 +1,32 @@
+package ch.epfl.bluebrain.nexus.delta.kernel.instances
+
+import io.circe.Json
+import munit.{FunSuite, Location}
+
+class ContentTypeInstancesSuite extends FunSuite with ContentTypeInstances {
+
+  def assertContentType(value: String, binary: Boolean, compressible: Boolean)(implicit l: Location) =
+    contentTypeDecoder.decodeJson(Json.fromString(value)) match {
+      case Left(err)          => fail(s"Could not decode $value: ${err.message}")
+      case Right(contentType) =>
+        assertEquals(contentType.binary, binary)
+        assertEquals(contentType.mediaType.comp.compressible, compressible)
+    }
+
+  test("'application/object-stream' is binary and not compressible") {
+    assertContentType("application/octet-stream", true, false)
+  }
+
+  test("'application/json' is not binary and compressible") {
+    assertContentType("application/json", false, true)
+  }
+
+  test("'application/cbor' is binary and compressible as it is registered as such in akka") {
+    assertContentType("application/cbor", true, true)
+  }
+
+  test("'application/nrrd' is binary and not compressible as it is a custom type") {
+    assertContentType("application/nrrd", true, false)
+  }
+
+}

delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileAttributes.scala

@@ -3,7 +3,6 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model
 import akka.http.scaladsl.model.Uri.Path
 import akka.http.scaladsl.model.{ContentType, Uri}
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin
-import ch.epfl.bluebrain.nexus.delta.plugins.storage.instances._
 import ch.epfl.bluebrain.nexus.delta.sdk.implicits._
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label
 import io.circe.{Decoder, Encoder}

delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileEvent.scala

@@ -10,7 +10,6 @@ import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.ContextValue
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords
 import ch.epfl.bluebrain.nexus.delta.sdk.implicits._
-import ch.epfl.bluebrain.nexus.delta.plugins.storage.instances._
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.ShowFileLocation
 import ch.epfl.bluebrain.nexus.delta.sdk.circe.JsonObjOps
 import ch.epfl.bluebrain.nexus.delta.sdk.jsonld.IriEncoder

delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileState.scala

@@ -5,7 +5,6 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageType
 import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri
 import ch.epfl.bluebrain.nexus.delta.sdk.model.{ResourceF, ResourceUris, Tags}
 import ch.epfl.bluebrain.nexus.delta.sdk.implicits._
-import ch.epfl.bluebrain.nexus.delta.plugins.storage.instances._
 import ch.epfl.bluebrain.nexus.delta.sourcing.Serializer
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.ResourceRef.Latest

delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala

@@ -277,7 +277,6 @@ object FilesRoutes {
 
   final case class LinkFile(filename: Option[String], mediaType: Option[ContentType], path: Path)
   object LinkFile {
-    import ch.epfl.bluebrain.nexus.delta.plugins.storage.instances._
     import ch.epfl.bluebrain.nexus.delta.rdf.instances._
     @nowarn("cat=unused")
     implicit private val config: Configuration      = Configuration.default.withStrictDecoding

delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/model/RemoteDiskStorageFileAttributes.scala

@@ -5,7 +5,7 @@ import cats.syntax.all._
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.{ComputedDigest, NotComputedDigest}
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.DigestAlgorithm
-import ch.epfl.bluebrain.nexus.delta.rdf.implicits._
+import ch.epfl.bluebrain.nexus.delta.sdk.implicits._
 import io.circe.{Decoder, DecodingFailure}
 import io.circe.generic.extras.Configuration
 import io.circe.generic.extras.semiauto._
@@ -55,9 +55,6 @@ object RemoteDiskStorageFileAttributes {
       }.flatten
     }
 
-    implicit val contentTypeDecoder: Decoder[ContentType] =
-      Decoder.decodeString.emap(ContentType.parse(_).left.map(_.mkString("\n")))
-
     deriveConfiguredDecoder[RemoteDiskStorageFileAttributes]
   }
 }

delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/implicits/package.scala

@@ -1,5 +1,6 @@
 package ch.epfl.bluebrain.nexus.delta.sdk
 
+import ch.epfl.bluebrain.nexus.delta.kernel.instances.ContentTypeInstances
 import ch.epfl.bluebrain.nexus.delta.kernel.syntax.{ClassTagSyntax, IOSyntax, InstantSyntax, KamonSyntax}
 import ch.epfl.bluebrain.nexus.delta.rdf.instances.{SecretInstances, TripleInstances, UriInstances}
 import ch.epfl.bluebrain.nexus.delta.rdf.syntax.{IriSyntax, IterableSyntax, JsonLdEncoderSyntax, JsonSyntax, PathSyntax, UriSyntax}
@@ -18,6 +19,7 @@ package object implicits
     with IdentityInstances
     with IriInstances
     with ProjectRefInstances
+    with ContentTypeInstances
     with JsonSyntax
     with IriSyntax
     with IriEncodingSyntax

delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/instances/package.scala

@@ -1,5 +1,6 @@
 package ch.epfl.bluebrain.nexus.delta.sdk
 
+import ch.epfl.bluebrain.nexus.delta.kernel.instances.ContentTypeInstances
 import ch.epfl.bluebrain.nexus.delta.rdf.instances.{SecretInstances, TripleInstances, UriInstances}
 
 package object instances
@@ -11,3 +12,4 @@ package object instances
     with UriInstances
     with SecretInstances
     with MediaTypeInstances
+    with ContentTypeInstances

storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/File.scala

@@ -44,9 +44,7 @@ object File {
     */
   final case class FileAttributes(location: Uri, bytes: Long, digest: Digest, mediaType: ContentType)
   object FileAttributes {
-    @nowarn("cat=unused")
-    implicit private val encMediaType: Encoder[ContentType] =
-      Encoder.encodeString.contramap(_.value)
+    import ch.epfl.bluebrain.nexus.delta.kernel.instances._
 
     @nowarn("cat=unused")
     implicit final private val uriDecoder: Decoder[Uri] =
@@ -56,10 +54,6 @@ object File {
     implicit final private val uriEncoder: Encoder[Uri] =
       Encoder.encodeString.contramap(_.toString())
 
-    @nowarn("cat=unused")
-    implicit private val decMediaType: Decoder[ContentType] =
-      Decoder.decodeString.emap(ContentType.parse(_).left.map(_.mkString("\n")))
-
     implicit val fileAttrEncoder: Encoder[FileAttributes] =
       deriveConfiguredEncoder[FileAttributes].mapJson(addContext(_, resourceCtxIri))
     implicit val fileAttrDecoder: Decoder[FileAttributes] = deriveConfiguredDecoder[FileAttributes]

tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/StorageSpec.scala

@@ -347,6 +347,23 @@ abstract class StorageSpec extends BaseIntegrationSpec {
     }
   }
 
+  "A custom binary file" should {
+    "not be downloadable compressed" in {
+      for {
+        _ <- filesDsl.uploadFile(customBinaryContent, projectRef, storageId, None)(expectCreated)
+        _ <- deltaClient
+               .get[ByteString](s"/files/$projectRef/attachment:${customBinaryContent.fileId}", Coyote, gzipHeaders) {
+                 filesDsl.expectFileContentAndMetadata(
+                   customBinaryContent.filename,
+                   customBinaryContent.ct,
+                   customBinaryContent.contents,
+                   compressed = false // the response should not be compressed despite the gzip headers
+                 )
+               }
+      } yield succeed
+    }
+  }
+
   "Deprecating a storage" should {
 
     "deprecate a storage" in {

tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/model/FileInput.scala

@@ -1,6 +1,7 @@
 package ch.epfl.bluebrain.nexus.tests.kg.files.model
 
-import akka.http.scaladsl.model.{ContentType, ContentTypes}
+import akka.http.scaladsl.model.MediaType.NotCompressible
+import akka.http.scaladsl.model.{ContentType, ContentTypes, MediaType}
 
 final case class FileInput(
     fileId: String,
@@ -42,4 +43,13 @@ object FileInput {
       "text file",
       Map("brainRegion" -> "hippocampus")
     )
+
+  val customBinaryContent =
+    FileInput(
+      "custom-binary",
+      "custom-binary",
+      ContentType.Binary(MediaType.applicationBinary("obj", NotCompressible)),
+      "text file",
+      Map("brainRegion" -> "hippocampus")
+    )
 }