From fe336011aa6ad01051ab0954da2e8b54c91ca84d Mon Sep 17 00:00:00 2001 From: Simon Dumas Date: Thu, 28 Sep 2023 11:54:34 +0200 Subject: [PATCH 1/9] Implement custom content type detection --- build.sbt | 27 +- .../kernel/http/MediaTypeDetectorConfig.scala | 41 +++ .../nexus/delta/kernel/utils/FileUtils.scala | 15 ++ .../http/MediaTypeDetectorConfigSuite.scala | 54 ++++ .../delta/kernel/utils/FileUtilsSuite.scala | 25 ++ .../storage/src/main/resources/storage.conf | 6 + .../delta/plugins/storage/files/Files.scala | 2 +- .../plugins/storage/files/FilesConfig.scala | 3 +- .../storage/files/FormDataExtractor.scala | 132 +++++---- .../plugins/storage/files/FilesSpec.scala | 3 +- .../storage/files/FormDataExtractorSpec.scala | 60 +++-- .../files/routes/FilesRoutesSpec.scala | 3 +- .../disk/DiskStorageSaveFileSpec.scala | 7 +- storage/src/main/resources/app.conf | 6 + .../epfl/bluebrain/nexus/storage/Main.scala | 6 +- .../bluebrain/nexus/storage/Storages.scala | 21 +- .../attributes/AttributesComputation.scala | 35 +-- .../attributes/ContentTypeDetector.scala | 45 ++++ .../nexus/storage/config/AppConfig.scala | 3 +- .../resources/content-type/file-example.json | 3 + .../test/resources/content-type/no-extension | 1 + .../nexus/storage/DiskStorageSpec.scala | 32 ++- .../bluebrain/nexus/storage/TarFlowSpec.scala | 5 +- .../AttributesComputationSpec.scala | 5 +- .../attributes/ContentTypeDetectorSuite.scala | 46 ++++ tests/docker/config/delta-postgres.conf | 8 + tests/docker/config/storage.conf | 34 +++ tests/docker/docker-compose.yml | 16 +- tests/src/test/resources/kg/files/file.custom | 1 + .../epfl/bluebrain/nexus/tests/BaseSpec.scala | 2 + .../bluebrain/nexus/tests/HttpClient.scala | 2 +- .../nexus/tests/kg/DiskStorageSpec.scala | 16 +- .../bluebrain/nexus/tests/kg/EventsSpec.scala | 4 +- .../nexus/tests/kg/MultiFetchSpec.scala | 2 +- .../nexus/tests/kg/ProjectsDeletionSpec.scala | 4 +- .../nexus/tests/kg/RemoteStorageSpec.scala | 71 ++--- .../nexus/tests/kg/S3StorageSpec.scala | 47 ++-- .../nexus/tests/kg/StorageSpec.scala | 251 +++++++----------- 38 files changed, 651 insertions(+), 393 deletions(-) create mode 100644 delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/http/MediaTypeDetectorConfig.scala create mode 100644 delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/FileUtils.scala create mode 100644 delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/http/MediaTypeDetectorConfigSuite.scala create mode 100644 delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/FileUtilsSuite.scala create mode 100644 storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/attributes/ContentTypeDetector.scala create mode 100644 storage/src/test/resources/content-type/file-example.json create mode 100644 storage/src/test/resources/content-type/no-extension create mode 100644 storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/attributes/ContentTypeDetectorSuite.scala create mode 100644 tests/docker/config/storage.conf create mode 100644 tests/src/test/resources/kg/files/file.custom diff --git a/build.sbt b/build.sbt index 609fe4a147..31dc221b39 100755 --- a/build.sbt +++ b/build.sbt @@ -21,7 +21,6 @@ val akkaCorsVersion = "1.2.0" val akkaVersion = "2.6.21" val alpakkaVersion = "3.0.4" val apacheCompressVersion = "1.24.0" -val apacheIoVersion = "1.3.2" val awsSdkVersion = "2.17.184" val byteBuddyAgentVersion = "1.10.17" val betterMonadicForVersion = "0.3.1" @@ -77,7 +76,6 @@ lazy val alpakkaFile = "com.lightbend.akka" %% "akka-stream-alp lazy val alpakkaSse = "com.lightbend.akka" %% "akka-stream-alpakka-sse" % alpakkaVersion lazy val alpakkaS3 = "com.lightbend.akka" %% "akka-stream-alpakka-s3" % alpakkaVersion lazy val apacheCompress = "org.apache.commons" % "commons-compress" % apacheCompressVersion -lazy val apacheIo = "org.apache.commons" % "commons-io" % apacheIoVersion lazy val awsSdk = "software.amazon.awssdk" % "s3" % awsSdkVersion lazy val betterMonadicFor = "com.olegpy" %% "better-monadic-for" % betterMonadicForVersion lazy val byteBuddyAgent = "net.bytebuddy" % "byte-buddy-agent" % byteBuddyAgentVersion @@ -206,6 +204,8 @@ lazy val kernel = project .settings(shared, compilation, coverage, release, assertJavaVersion) .settings( libraryDependencies ++= Seq( + akkaActorTyped, // Needed to create content type + akkaHttpCore, caffeine, catsRetry, circeCore, @@ -216,6 +216,7 @@ lazy val kernel = project log4cats, pureconfig, scalaLogging, + munit % Test, scalaTest % Test ), addCompilerPlugin(kindProjector), @@ -733,29 +734,15 @@ lazy val storage = project servicePackaging, coverageMinimumStmtTotal := 75 ) - .settings(cargo := { - import scala.sys.process._ - - val log = streams.value.log - val cmd = Process(Seq("cargo", "build", "--release"), baseDirectory.value / "permissions-fixer") - if (cmd.! == 0) { - log.success("Cargo build successful.") - (baseDirectory.value / "permissions-fixer" / "target" / "release" / "nexus-fixer") -> "bin/nexus-fixer" - } else { - log.error("Cargo build failed.") - throw new RuntimeException - } - }) + .dependsOn(kernel) .settings( name := "storage", moduleName := "storage", buildInfoKeys := Seq[BuildInfoKey](version), buildInfoPackage := "ch.epfl.bluebrain.nexus.storage.config", Docker / packageName := "nexus-storage", - javaSpecificationVersion := "1.8", libraryDependencies ++= Seq( apacheCompress, - apacheIo, akkaHttp, akkaHttpCirce, akkaStream, @@ -772,6 +759,7 @@ lazy val storage = project akkaHttpTestKit % Test, akkaTestKit % Test, mockito % Test, + munit % Test, scalaTest % Test ), cleanFiles ++= Seq( @@ -779,10 +767,7 @@ lazy val storage = project baseDirectory.value / "nexus-storage.jar" ), Test / testOptions += Tests.Argument(TestFrameworks.ScalaTest, "-o", "-u", "target/test-reports"), - Test / parallelExecution := false, - Universal / mappings := { - (Universal / mappings).value :+ cargo.value - } + Test / parallelExecution := false ) lazy val tests = project diff --git a/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/http/MediaTypeDetectorConfig.scala b/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/http/MediaTypeDetectorConfig.scala new file mode 100644 index 0000000000..354fbc6e17 --- /dev/null +++ b/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/http/MediaTypeDetectorConfig.scala @@ -0,0 +1,41 @@ +package ch.epfl.bluebrain.nexus.delta.kernel.http + +import akka.http.scaladsl.model.MediaType +import cats.syntax.all._ +import pureconfig.ConfigReader +import pureconfig.configurable.genericMapReader +import pureconfig.error.CannotConvert + +/** + * Allows to define custom media types for the given extensions + */ +final case class MediaTypeDetectorConfig(extensions: Map[String, MediaType]) { + def find(extension: String): Option[MediaType] = extensions.get(extension) + +} + +object MediaTypeDetectorConfig { + + val Empty = new MediaTypeDetectorConfig(Map.empty) + + def apply(values: (String, MediaType)*) = new MediaTypeDetectorConfig(values.toMap) + + implicit final val mediaTypeDetectorConfigReader: ConfigReader[MediaTypeDetectorConfig] = { + implicit val mediaTypeConfigReader: ConfigReader[MediaType] = + ConfigReader.fromString(str => + MediaType + .parse(str) + .leftMap(_ => CannotConvert(str, classOf[MediaType].getSimpleName, s"'$str' is not a valid content type.")) + ) + implicit val mapReader: ConfigReader[Map[String, MediaType]] = genericMapReader(Right(_)) + + ConfigReader.fromCursor { cursor => + for { + obj <- cursor.asObjectCursor + extensionsKey <- obj.atKey("extensions") + extensions <- ConfigReader[Map[String, MediaType]].from(extensionsKey) + } yield MediaTypeDetectorConfig(extensions) + } + } + +} diff --git a/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/FileUtils.scala b/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/FileUtils.scala new file mode 100644 index 0000000000..e041b4b1c4 --- /dev/null +++ b/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/FileUtils.scala @@ -0,0 +1,15 @@ +package ch.epfl.bluebrain.nexus.delta.kernel.utils + +object FileUtils { + + /** + * Extracts the extension from the given filename + */ + def extension(filename: String): Option[String] = { + val lastDotIndex = filename.lastIndexOf('.') + Option.when(lastDotIndex >= 0) { + filename.substring(lastDotIndex + 1) + } + } + +} diff --git a/delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/http/MediaTypeDetectorConfigSuite.scala b/delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/http/MediaTypeDetectorConfigSuite.scala new file mode 100644 index 0000000000..adde258bb8 --- /dev/null +++ b/delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/http/MediaTypeDetectorConfigSuite.scala @@ -0,0 +1,54 @@ +package ch.epfl.bluebrain.nexus.delta.kernel.http + +import akka.http.scaladsl.model.ContentTypes +import munit.FunSuite +import pureconfig.ConfigSource + +class MediaTypeDetectorConfigSuite extends FunSuite { + + private def parseConfig(value: String) = + ConfigSource.string(value).at("media-type-detector").load[MediaTypeDetectorConfig] + + test("Parse successfully the config with no defined extension") { + val config = parseConfig( + """ + |media-type-detector { + | extensions { + | } + |} + |""".stripMargin + ) + + val expected = MediaTypeDetectorConfig.Empty + assertEquals(config, Right(expected)) + } + + test("Parse successfully the config") { + val config = parseConfig( + """ + |media-type-detector { + | extensions { + | json = application/json + | } + |} + |""".stripMargin + ) + + val expected = MediaTypeDetectorConfig("json" -> ContentTypes.`application/json`.mediaType) + assertEquals(config, Right(expected)) + } + + test("Fail to parse the config with an invalid content type") { + val config = parseConfig( + """ + |media-type-detector { + | extensions { + | json = xxx + | } + |} + |""".stripMargin + ) + + assert(config.isLeft, "Parsing must fail with an invalid content type") + } +} diff --git a/delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/FileUtilsSuite.scala b/delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/FileUtilsSuite.scala new file mode 100644 index 0000000000..e34280fbae --- /dev/null +++ b/delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/FileUtilsSuite.scala @@ -0,0 +1,25 @@ +package ch.epfl.bluebrain.nexus.delta.kernel.utils + +import munit.FunSuite + +class FileUtilsSuite extends FunSuite { + + test("Detect json extension") { + val obtained = FileUtils.extension("my-file.json") + val expected = Some("json") + assertEquals(obtained, expected) + } + + test("Detect zip extension") { + val obtained = FileUtils.extension("my-file.json.zip") + val expected = Some("zip") + assertEquals(obtained, expected) + } + + test("Detect no extension") { + val obtained = FileUtils.extension("my-file") + val expected = None + assertEquals(obtained, expected) + } + +} diff --git a/delta/plugins/storage/src/main/resources/storage.conf b/delta/plugins/storage/src/main/resources/storage.conf index 6f159cfd84..ad5968955f 100644 --- a/delta/plugins/storage/src/main/resources/storage.conf +++ b/delta/plugins/storage/src/main/resources/storage.conf @@ -76,6 +76,12 @@ plugins.storage { files { # the files event log configuration event-log = ${app.defaults.event-log} + + media-type-detector { + extensions { + #extension = "application/custom" + } + } } defaults { # the name of the default storage diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala index 5b2acf3320..48a69042ad 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala @@ -720,7 +720,7 @@ object Files { ): Files = { implicit val classicAs: ClassicActorSystem = as.classicSystem new Files( - FormDataExtractor.apply, + FormDataExtractor(config.mediaTypeDetector), ScopedEventLog(definition, config.eventLog, xas), aclCheck, fetchContext, diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesConfig.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesConfig.scala index 5ed705285f..e7ead7d95b 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesConfig.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesConfig.scala @@ -1,5 +1,6 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files +import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig import pureconfig.ConfigReader import pureconfig.generic.semiauto.deriveReader @@ -10,7 +11,7 @@ import pureconfig.generic.semiauto.deriveReader * @param eventLog * configuration of the event log */ -final case class FilesConfig(eventLog: EventLogConfig) +final case class FilesConfig(eventLog: EventLogConfig, mediaTypeDetector: MediaTypeDetectorConfig) object FilesConfig { implicit final val filesConfigReader: ConfigReader[FilesConfig] = diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala index 7540d384c5..6787e50bf1 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala @@ -1,13 +1,15 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files import akka.actor.ActorSystem +import akka.http.scaladsl.model.MediaTypes.`multipart/form-data` import akka.http.scaladsl.model._ import akka.http.scaladsl.server._ import akka.http.scaladsl.unmarshalling.Unmarshaller.UnsupportedContentTypeException -import akka.http.scaladsl.unmarshalling.{FromEntityUnmarshaller, Unmarshaller} +import akka.http.scaladsl.unmarshalling.{FromEntityUnmarshaller, MultipartUnmarshallers, Unmarshaller} import akka.stream.scaladsl.{Keep, Sink} import cats.syntax.all._ -import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF +import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig +import ch.epfl.bluebrain.nexus.delta.kernel.utils.{FileUtils, UUIDF} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection.{FileTooLarge, InvalidMultipartFieldName, WrappedAkkaRejection} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileDescription, FileRejection} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri @@ -41,55 +43,81 @@ object FormDataExtractor { private val fieldName: String = "file" - def apply(implicit - uuidF: UUIDF, - as: ActorSystem, - sc: Scheduler, - um: FromEntityUnmarshaller[Multipart.FormData] - ): FormDataExtractor = new FormDataExtractor { - override def apply( - id: Iri, - entity: HttpEntity, - maxFileSize: Long, - storageAvailableSpace: Option[Long] - ): IO[FileRejection, (FileDescription, BodyPartEntity)] = { - val sizeLimit = Math.min(storageAvailableSpace.getOrElse(Long.MaxValue), maxFileSize) - IO.deferFuture(um(entity.withSizeLimit(sizeLimit))) - .mapError { - case RejectionError(r) => - WrappedAkkaRejection(r) - case Unmarshaller.NoContentException => - WrappedAkkaRejection(RequestEntityExpectedRejection) - case x: UnsupportedContentTypeException => - WrappedAkkaRejection(UnsupportedRequestContentTypeRejection(x.supported, x.actualContentType)) - case x: IllegalArgumentException => - WrappedAkkaRejection(ValidationRejection(Option(x.getMessage).getOrElse(""), Some(x))) - case x: ExceptionWithErrorInfo => - WrappedAkkaRejection(MalformedRequestContentRejection(x.info.format(withDetail = false), x)) - case x => - WrappedAkkaRejection(MalformedRequestContentRejection(Option(x.getMessage).getOrElse(""), x)) - } - .flatMap { formData => - IO.fromFuture( - formData.parts - .mapAsync(parallelism = 1) { - case part if part.name == fieldName => - FileDescription(part.filename.getOrElse("file"), part.entity.contentType).runToFuture.map { desc => - Some(desc -> part.entity) - } - case part => - part.entity.discardBytes().future.as(None) - } - .collect { case Some(values) => values } - .toMat(Sink.headOption)(Keep.right) - .run() - ).mapError { - case _: EntityStreamSizeException => - FileTooLarge(maxFileSize, storageAvailableSpace) - case th => - WrappedAkkaRejection(MalformedRequestContentRejection(th.getMessage, th)) - }.flatMap(IO.fromOption(_, InvalidMultipartFieldName(id))) - } + private val defaultContentType: ContentType.Binary = ContentTypes.`application/octet-stream` + + // Creating an unmarshaller defaulting to `application/octet-stream` as a content type + implicit private val um: FromEntityUnmarshaller[Multipart.FormData] = + MultipartUnmarshallers + .multipartUnmarshaller[Multipart.FormData, Multipart.FormData.BodyPart, Multipart.FormData.BodyPart.Strict]( + mediaRange = `multipart/form-data`, + defaultContentType = defaultContentType, + createBodyPart = (entity, headers) => Multipart.General.BodyPart(entity, headers).toFormDataBodyPart.get, + createStreamed = (_, parts) => Multipart.FormData(parts), + createStrictBodyPart = + (entity, headers) => Multipart.General.BodyPart.Strict(entity, headers).toFormDataBodyPart.get, + createStrict = (_, parts) => Multipart.FormData.Strict(parts) + ) + + def apply( + mediaTypeDetector: MediaTypeDetectorConfig + )(implicit uuidF: UUIDF, as: ActorSystem, sc: Scheduler): FormDataExtractor = + new FormDataExtractor { + override def apply( + id: Iri, + entity: HttpEntity, + maxFileSize: Long, + storageAvailableSpace: Option[Long] + ): IO[FileRejection, (FileDescription, BodyPartEntity)] = { + val sizeLimit = Math.min(storageAvailableSpace.getOrElse(Long.MaxValue), maxFileSize) + IO.deferFuture(um(entity.withSizeLimit(sizeLimit))) + .mapError { + case RejectionError(r) => + WrappedAkkaRejection(r) + case Unmarshaller.NoContentException => + WrappedAkkaRejection(RequestEntityExpectedRejection) + case x: UnsupportedContentTypeException => + WrappedAkkaRejection(UnsupportedRequestContentTypeRejection(x.supported, x.actualContentType)) + case x: IllegalArgumentException => + WrappedAkkaRejection(ValidationRejection(Option(x.getMessage).getOrElse(""), Some(x))) + case x: ExceptionWithErrorInfo => + WrappedAkkaRejection(MalformedRequestContentRejection(x.info.format(withDetail = false), x)) + case x => + WrappedAkkaRejection(MalformedRequestContentRejection(Option(x.getMessage).getOrElse(""), x)) + } + .flatMap { formData => + IO.fromFuture( + formData.parts + .mapAsync(parallelism = 1) { + case part if part.name == fieldName => + val filename = part.filename.getOrElse("file") + val contentType = detectContentType(filename, part.entity.contentType) + FileDescription(filename, contentType).runToFuture.map { desc => + Some(desc -> part.entity) + } + case part => + part.entity.discardBytes().future.as(None) + } + .collect { case Some(values) => values } + .toMat(Sink.headOption)(Keep.right) + .run() + ).mapError { + case _: EntityStreamSizeException => + FileTooLarge(maxFileSize, storageAvailableSpace) + case th => + WrappedAkkaRejection(MalformedRequestContentRejection(th.getMessage, th)) + }.flatMap(IO.fromOption(_, InvalidMultipartFieldName(id))) + } + } + + private def detectContentType(filename: String, contentTypeFromAkka: ContentType) = { + val bodyDefinedContentType = Option.when(contentTypeFromAkka != defaultContentType)(contentTypeFromAkka) + + def detectFromConfig = for { + extension <- FileUtils.extension(filename) + customMediaType <- mediaTypeDetector.find(extension) + } yield ContentType(customMediaType, () => HttpCharsets.`UTF-8`) + + bodyDefinedContentType.orElse(detectFromConfig).getOrElse(contentTypeFromAkka) + } } - } } diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala index 9b01a64594..39174a9c6e 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala @@ -5,6 +5,7 @@ import akka.actor.{typed, ActorSystem} import akka.http.scaladsl.model.ContentTypes.`text/plain(UTF-8)` import akka.http.scaladsl.model.Uri import akka.testkit.TestKit +import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig import ch.epfl.bluebrain.nexus.delta.plugins.storage.RemoteContextResolutionFixture import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.NotComputedDigest import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Storage @@ -121,7 +122,7 @@ class FilesSpec(docker: RemoteStorageDocker) storageStatistics, xas, cfg, - FilesConfig(eventLogConfig), + FilesConfig(eventLogConfig, MediaTypeDetectorConfig.Empty), remoteDiskStorageClient ) diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractorSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractorSpec.scala index c5a1be42d6..5985d04e6d 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractorSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractorSpec.scala @@ -1,15 +1,16 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files import akka.actor.ActorSystem -import akka.http.scaladsl.model.ContentTypes.`text/plain(UTF-8)` -import akka.http.scaladsl.model.{HttpEntity, Multipart} +import akka.http.scaladsl.model.ContentTypes._ +import akka.http.scaladsl.model.{ContentType, HttpCharsets, HttpEntity, MediaType, Multipart} import akka.testkit.TestKit +import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileDescription import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection.{FileTooLarge, InvalidMultipartFieldName} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.AkkaSourceHelpers import ch.epfl.bluebrain.nexus.delta.sdk.syntax._ -import ch.epfl.bluebrain.nexus.testkit.IOValues +import ch.epfl.bluebrain.nexus.testkit.{EitherValuable, IOValues} import monix.execution.Scheduler import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpecLike @@ -21,6 +22,7 @@ class FormDataExtractorSpec with AnyWordSpecLike with Matchers with IOValues + with EitherValuable with AkkaSourceHelpers { "A Form Data HttpEntity" should { @@ -32,37 +34,51 @@ class FormDataExtractorSpec val content = "file content" val iri = iri"http://localhost/file" - val extractor = FormDataExtractor.apply + val customMediaType = MediaType.parse("application/custom").rightValue + val customContentType = ContentType(customMediaType, () => HttpCharsets.`UTF-8`) + val mediaTypeDetector = MediaTypeDetectorConfig(Map("custom" -> customMediaType)) + val extractor = FormDataExtractor(mediaTypeDetector) - "be extracted" in { - val entity = - Multipart - .FormData( - Multipart.FormData - .BodyPart("file", HttpEntity(`text/plain(UTF-8)`, content), Map("filename" -> "file.txt")) - ) - .toEntity() + def createEntity(bodyPart: String, contentType: ContentType, filename: Option[String]) = + Multipart + .FormData( + Multipart.FormData + .BodyPart(bodyPart, HttpEntity(contentType, content.getBytes), filename.map("filename" -> _).toMap) + ) + .toEntity() - val expectedDescription = FileDescription(uuid, "file.txt", Some(`text/plain(UTF-8)`)) + "be extracted with the default content type" in { + val entity = createEntity("file", NoContentType, Some("file.txt")) + + val expectedDescription = FileDescription(uuid, "file.txt", Some(`application/octet-stream`)) val (description, resultEntity) = extractor(iri, entity, 179, None).accepted description shouldEqual expectedDescription consume(resultEntity.dataBytes) shouldEqual content } - "fail to be extracted if no file part exists found" in { - val entity = - Multipart - .FormData(Multipart.FormData.BodyPart("other", HttpEntity(`text/plain(UTF-8)`, content), Map.empty)) - .toEntity() + "be extracted with the custom media type" in { + val entity = createEntity("file", NoContentType, Some("file.custom")) + val expectedDescription = FileDescription(uuid, "file.custom", Some(customContentType)) + val (description, resultEntity) = extractor(iri, entity, 2000, None).accepted + description shouldEqual expectedDescription + consume(resultEntity.dataBytes) shouldEqual content + } + "be extracted with the provided content type header" in { + val entity = createEntity("file", `text/plain(UTF-8)`, Some("file.custom")) + val expectedDescription = FileDescription(uuid, "file.custom", Some(`text/plain(UTF-8)`)) + val (description, resultEntity) = extractor(iri, entity, 2000, None).accepted + description shouldEqual expectedDescription + consume(resultEntity.dataBytes) shouldEqual content + } + + "fail to be extracted if no file part exists found" in { + val entity = createEntity("other", NoContentType, None) extractor(iri, entity, 179, None).rejectedWith[InvalidMultipartFieldName] } "fail to be extracted if payload size is too large" in { - val entity = - Multipart - .FormData(Multipart.FormData.BodyPart("other", HttpEntity(`text/plain(UTF-8)`, content), Map.empty)) - .toEntity() + val entity = createEntity("other", `text/plain(UTF-8)`, None) extractor(iri, entity, 10, None).rejected shouldEqual FileTooLarge(10L, None) } diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala index b22a4106d8..638e7ba20f 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala @@ -7,6 +7,7 @@ import akka.http.scaladsl.model.MediaTypes.`text/html` import akka.http.scaladsl.model.headers.{Accept, Location, OAuth2BearerToken} import akka.http.scaladsl.model.{StatusCodes, Uri} import akka.http.scaladsl.server.Route +import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.ComputedDigest import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileRejection} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.FilesRoutesSpec.fileMetadata @@ -114,7 +115,7 @@ class FilesRoutesSpec storagesStatistics, xas, config, - FilesConfig(eventLogConfig), + FilesConfig(eventLogConfig, MediaTypeDetectorConfig.Empty), remoteDiskStorageClient ) private val groupDirectives = diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFileSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFileSpec.scala index 3ba8770ce3..c38b86e8ce 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFileSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFileSpec.scala @@ -20,13 +20,13 @@ import ch.epfl.bluebrain.nexus.testkit.remotestorage.RemoteStorageDocker import ch.epfl.bluebrain.nexus.testkit.{EitherValuable, IOValues} import io.circe.Json import monix.execution.Scheduler -import org.apache.commons.io.FileUtils import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpecLike import java.nio.file.{Files, Paths} import java.util.UUID +import scala.reflect.io.Directory class DiskStorageSaveFileSpec extends TestKit(ActorSystem("DiskStorageSaveFileSpec")) @@ -80,5 +80,8 @@ class DiskStorageSaveFileSpec } } - override protected def afterAll(): Unit = FileUtils.deleteDirectory(volume.value.toFile) + override protected def afterAll(): Unit = { + Directory(volume.value.toFile).deleteRecursively() + () + } } diff --git a/storage/src/main/resources/app.conf b/storage/src/main/resources/app.conf index 2563e78f59..db55274922 100644 --- a/storage/src/main/resources/app.conf +++ b/storage/src/main/resources/app.conf @@ -43,6 +43,12 @@ app { fixer-command = [] } + media-type-detector { + extensions { + #extension = "application/custom" + } + } + digest { # the digest algorithm algorithm = "SHA-256" diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Main.scala b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Main.scala index b2a7f9afc8..8257a919cc 100644 --- a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Main.scala +++ b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Main.scala @@ -2,7 +2,6 @@ package ch.epfl.bluebrain.nexus.storage import java.nio.file.Paths import java.time.Clock - import akka.actor.ActorSystem import akka.event.{Logging, LoggingAdapter} import akka.http.scaladsl.Http @@ -10,7 +9,7 @@ import akka.http.scaladsl.server.Route import akka.util.Timeout import cats.effect.Effect import ch.epfl.bluebrain.nexus.storage.Storages.DiskStorage -import ch.epfl.bluebrain.nexus.storage.attributes.AttributesCache +import ch.epfl.bluebrain.nexus.storage.attributes.{AttributesCache, ContentTypeDetector} import ch.epfl.bluebrain.nexus.storage.config.{AppConfig, Settings} import ch.epfl.bluebrain.nexus.storage.config.AppConfig._ import ch.epfl.bluebrain.nexus.storage.routes.Routes @@ -60,9 +59,10 @@ object Main { implicit val deltaIdentities: DeltaIdentitiesClient[Task] = new DeltaIdentitiesClient[Task](appConfig.delta) implicit val timeout = Timeout(1.minute) implicit val clock = Clock.systemUTC + implicit val contentTypeDetector = new ContentTypeDetector(appConfig.mediaTypeDetector) val storages: Storages[Task, AkkaSource] = - new DiskStorage(appConfig.storage, appConfig.digest, AttributesCache[Task, AkkaSource]) + new DiskStorage(appConfig.storage, contentTypeDetector, appConfig.digest, AttributesCache[Task, AkkaSource]) val logger: LoggingAdapter = Logging(as, getClass) diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Storages.scala b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Storages.scala index 611ca9edab..164b192f58 100644 --- a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Storages.scala +++ b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Storages.scala @@ -1,9 +1,5 @@ package ch.epfl.bluebrain.nexus.storage -import java.net.URLDecoder -import java.nio.file.StandardCopyOption._ -import java.nio.file.{Files, Path, Paths} -import java.security.MessageDigest import akka.http.scaladsl.model.Uri import akka.stream.Materializer import akka.stream.alpakka.file.scaladsl.Directory @@ -16,10 +12,14 @@ import ch.epfl.bluebrain.nexus.storage.StorageError.{InternalError, PathInvalid, import ch.epfl.bluebrain.nexus.storage.Storages.BucketExistence._ import ch.epfl.bluebrain.nexus.storage.Storages.PathExistence._ import ch.epfl.bluebrain.nexus.storage.Storages.{BucketExistence, PathExistence} -import ch.epfl.bluebrain.nexus.storage.attributes.AttributesCache import ch.epfl.bluebrain.nexus.storage.attributes.AttributesComputation._ +import ch.epfl.bluebrain.nexus.storage.attributes.{AttributesCache, ContentTypeDetector} import ch.epfl.bluebrain.nexus.storage.config.AppConfig.{DigestConfig, StorageConfig} +import java.net.URLDecoder +import java.nio.file.StandardCopyOption._ +import java.nio.file.{Files, Path, Paths} +import java.security.MessageDigest import scala.annotation.tailrec import scala.concurrent.{ExecutionContext, Future} import scala.sys.process._ @@ -150,7 +150,12 @@ object Storages { /** * An Disk implementation of Storage interface. */ - final class DiskStorage[F[_]](config: StorageConfig, digestConfig: DigestConfig, cache: AttributesCache[F])(implicit + final class DiskStorage[F[_]]( + config: StorageConfig, + contentTypeDetector: ContentTypeDetector, + digestConfig: DigestConfig, + cache: AttributesCache[F] + )(implicit ec: ExecutionContext, mt: Materializer, F: Effect[F] @@ -197,7 +202,7 @@ object Storages { .toMat(FileIO.toPath(absFilePath)) { case (digFuture, ioFuture) => digFuture.zipWith(ioFuture) { case (digest, io) if absFilePath.toFile.exists() => - Future(FileAttributes(absFilePath.toAkkaUri, io.count, digest, detectMediaType(absFilePath))) + Future(FileAttributes(absFilePath.toAkkaUri, io.count, digest, contentTypeDetector(absFilePath))) case _ => Future.failed(InternalError(s"I/O error writing file to path '$path'")) } @@ -240,7 +245,7 @@ object Storages { } def computeSizeAndMove(isDir: Boolean): F[RejOrAttributes] = { - lazy val mediaType = detectMediaType(absDestPath, isDir) + lazy val mediaType = contentTypeDetector(absDestPath, isDir) size(absSourcePath).flatMap { computedSize => F.fromTry(Try(Files.createDirectories(absDestPath.getParent))) >> F.fromTry(Try(Files.move(absSourcePath, absDestPath, ATOMIC_MOVE))) >> diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/attributes/AttributesComputation.scala b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/attributes/AttributesComputation.scala index d91b5cda20..b33f0dd71e 100644 --- a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/attributes/AttributesComputation.scala +++ b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/attributes/AttributesComputation.scala @@ -1,11 +1,5 @@ package ch.epfl.bluebrain.nexus.storage.attributes -import java.nio.file.{Files, Path} -import java.security.MessageDigest - -import akka.http.scaladsl.model.HttpCharsets.`UTF-8` -import akka.http.scaladsl.model.MediaTypes.{`application/octet-stream`, `application/x-tar`} -import akka.http.scaladsl.model.{ContentType, MediaType, MediaTypes} import akka.stream.Materializer import akka.stream.scaladsl.{Keep, Sink} import akka.util.ByteString @@ -14,8 +8,9 @@ import cats.implicits._ import ch.epfl.bluebrain.nexus.storage.File.{Digest, FileAttributes} import ch.epfl.bluebrain.nexus.storage.StorageError.InternalError import ch.epfl.bluebrain.nexus.storage._ -import org.apache.commons.io.FilenameUtils +import java.nio.file.{Files, Path} +import java.security.MessageDigest import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Success, Try} @@ -36,28 +31,7 @@ trait AttributesComputation[F[_], Source] { object AttributesComputation { - /** - * Detects the media type of the provided path, based on the file system detector available for a certain path or on - * the path extension. If the path is a directory, a application/x-tar content-type is returned - * - * @param path - * the path - * @param isDir - * flag to decide whether or not the path is a directory - */ - def detectMediaType(path: Path, isDir: Boolean = false): ContentType = - if (isDir) { - `application/x-tar` - } else { - lazy val fromExtension = Try(MediaTypes.forExtension(FilenameUtils.getExtension(path.toFile.getName))) - .getOrElse(`application/octet-stream`) - val mediaType: MediaType = Try(Files.probeContentType(path)) match { - case Success(value) if value != null && value.nonEmpty => MediaType.parse(value).getOrElse(fromExtension) - case _ => fromExtension - } - ContentType(mediaType, () => `UTF-8`) - } - private def sinkSize: Sink[ByteString, Future[Long]] = Sink.fold(0L)(_ + _.size) + private def sinkSize: Sink[ByteString, Future[Long]] = Sink.fold(0L)(_ + _.size) def sinkDigest(msgDigest: MessageDigest)(implicit ec: ExecutionContext): Sink[ByteString, Future[Digest]] = Sink @@ -76,6 +50,7 @@ object AttributesComputation { * a AttributesComputation implemented for a source of type AkkaSource */ implicit def akkaAttributes[F[_]](implicit + contentTypeDetector: ContentTypeDetector, ec: ExecutionContext, mt: Materializer, F: Effect[F] @@ -91,7 +66,7 @@ object AttributesComputation { .alsoToMat(sinkSize)(Keep.right) .toMat(sinkDigest(msgDigest)) { (bytesF, digestF) => (bytesF, digestF).mapN { case (bytes, digest) => - FileAttributes(path.toAkkaUri, bytes, digest, detectMediaType(path, isDir)) + FileAttributes(path.toAkkaUri, bytes, digest, contentTypeDetector(path, isDir)) } } .run() diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/attributes/ContentTypeDetector.scala b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/attributes/ContentTypeDetector.scala new file mode 100644 index 0000000000..cc6050fb83 --- /dev/null +++ b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/attributes/ContentTypeDetector.scala @@ -0,0 +1,45 @@ +package ch.epfl.bluebrain.nexus.storage.attributes + +import akka.http.scaladsl.model.{ContentType, MediaType, MediaTypes} +import akka.http.scaladsl.model.HttpCharsets.`UTF-8` +import akka.http.scaladsl.model.MediaTypes.{`application/octet-stream`, `application/x-tar`} +import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig +import ch.epfl.bluebrain.nexus.delta.kernel.utils.FileUtils + +import java.nio.file.{Files, Path} +import scala.util.Try + +final class ContentTypeDetector(config: MediaTypeDetectorConfig) { + + /** + * Detects the media type of the provided path, based on the custom detector, the file system detector available for + * a certain path or on the path extension. If the path is a directory, a application/x-tar content-type is returned + * + * @param path + * the path + * @param isDir + * flag to decide whether or not the path is a directory + */ + def apply(path: Path, isDir: Boolean = false): ContentType = + if (isDir) { + `application/x-tar` + } else { + val extension = FileUtils.extension(path.toFile.getName) + + val customDetector = extension.flatMap(config.find) + + def fileContentDetector = + for { + probed <- Try(Files.probeContentType(path)).toOption + rawContentType <- Option.when(probed != null && probed.nonEmpty)(probed) + parsedContentType <- MediaType.parse(rawContentType).toOption + } yield parsedContentType + + def defaultAkkaDetector = extension.flatMap { e => Try(MediaTypes.forExtension(e)).toOption } + + val mediaType = + customDetector.orElse(fileContentDetector).orElse(defaultAkkaDetector).getOrElse(`application/octet-stream`) + ContentType(mediaType, () => `UTF-8`) + } + +} diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/config/AppConfig.scala b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/config/AppConfig.scala index e1120b4bcf..4c20aa4abe 100644 --- a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/config/AppConfig.scala +++ b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/config/AppConfig.scala @@ -1,8 +1,8 @@ package ch.epfl.bluebrain.nexus.storage.config import java.nio.file.Path - import akka.http.scaladsl.model.Uri +import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig import ch.epfl.bluebrain.nexus.storage.DeltaIdentitiesClient.Identity.{Anonymous, Subject, User} import ch.epfl.bluebrain.nexus.storage.JsonLdCirceSupport.OrderedKeys import ch.epfl.bluebrain.nexus.storage.config.AppConfig._ @@ -31,6 +31,7 @@ final case class AppConfig( storage: StorageConfig, subject: SubjectConfig, delta: DeltaClientConfig, + mediaTypeDetector: MediaTypeDetectorConfig, digest: DigestConfig ) diff --git a/storage/src/test/resources/content-type/file-example.json b/storage/src/test/resources/content-type/file-example.json new file mode 100644 index 0000000000..b12b0e3908 --- /dev/null +++ b/storage/src/test/resources/content-type/file-example.json @@ -0,0 +1,3 @@ +{ + "content": "Example" +} \ No newline at end of file diff --git a/storage/src/test/resources/content-type/no-extension b/storage/src/test/resources/content-type/no-extension new file mode 100644 index 0000000000..6b584e8ece --- /dev/null +++ b/storage/src/test/resources/content-type/no-extension @@ -0,0 +1 @@ +content \ No newline at end of file diff --git a/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/DiskStorageSpec.scala b/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/DiskStorageSpec.scala index ba727450a0..e66ab73dbb 100644 --- a/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/DiskStorageSpec.scala +++ b/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/DiskStorageSpec.scala @@ -1,9 +1,5 @@ package ch.epfl.bluebrain.nexus.storage -import java.io.File -import java.nio.charset.StandardCharsets -import java.nio.charset.StandardCharsets.UTF_8 -import java.nio.file.{Files, Paths} import akka.actor.ActorSystem import akka.http.scaladsl.model.ContentTypes._ import akka.http.scaladsl.model.MediaTypes.`application/x-tar` @@ -12,23 +8,28 @@ import akka.stream.scaladsl.{Sink, Source} import akka.testkit.TestKit import akka.util.ByteString import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig import ch.epfl.bluebrain.nexus.storage.File.{Digest, FileAttributes} import ch.epfl.bluebrain.nexus.storage.Rejection.{PathAlreadyExists, PathNotFound} import ch.epfl.bluebrain.nexus.storage.StorageError.{PathInvalid, PermissionsFixingFailed} import ch.epfl.bluebrain.nexus.storage.Storages.BucketExistence.{BucketDoesNotExist, BucketExists} import ch.epfl.bluebrain.nexus.storage.Storages.DiskStorage import ch.epfl.bluebrain.nexus.storage.Storages.PathExistence.{PathDoesNotExist, PathExists} -import ch.epfl.bluebrain.nexus.storage.attributes.AttributesCache +import ch.epfl.bluebrain.nexus.storage.attributes.{AttributesCache, ContentTypeDetector} import ch.epfl.bluebrain.nexus.storage.config.AppConfig.{DigestConfig, StorageConfig} import ch.epfl.bluebrain.nexus.storage.utils.{EitherValues, IOEitherValues, Randomness} -import org.apache.commons.io.FileUtils import org.mockito.IdiomaticMockito import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpecLike import org.scalatest.{BeforeAndAfterAll, Inspectors, OptionValues} +import java.io.File +import java.nio.charset.StandardCharsets +import java.nio.charset.StandardCharsets.UTF_8 +import java.nio.file.{Files, Paths} import scala.concurrent.ExecutionContext import scala.concurrent.duration._ +import scala.reflect.io.Directory class DiskStorageSpec extends TestKit(ActorSystem("DiskStorageSpec")) @@ -46,15 +47,17 @@ class DiskStorageSpec implicit val ec: ExecutionContext = system.dispatcher - val rootPath = Files.createTempDirectory("storage-test") - val scratchPath = Files.createTempDirectory("scratch") - val sConfig = StorageConfig(rootPath, List(scratchPath), Paths.get("nexus"), fixerEnabled = true, Vector("/bin/echo")) - val dConfig = DigestConfig("SHA-256", 1L, 1, 1, 1.second) - val cache = mock[AttributesCache[IO]] - val storage = new DiskStorage[IO](sConfig, dConfig, cache) + val rootPath = Files.createTempDirectory("storage-test") + val scratchPath = Files.createTempDirectory("scratch") + val sConfig = StorageConfig(rootPath, List(scratchPath), Paths.get("nexus"), fixerEnabled = true, Vector("/bin/echo")) + val dConfig = DigestConfig("SHA-256", 1L, 1, 1, 1.second) + val contentTypeDetector = new ContentTypeDetector(MediaTypeDetectorConfig.Empty) + val cache = mock[AttributesCache[IO]] + val storage = new DiskStorage[IO](sConfig, contentTypeDetector, dConfig, cache) override def afterAll(): Unit = { - FileUtils.deleteDirectory(rootPath.toFile) + Directory(rootPath.toFile).deleteRecursively() + () } trait AbsoluteDirectoryCreated { @@ -164,7 +167,8 @@ class DiskStorageSpec "fail when call to nexus-fixer fails" in new AbsoluteDirectoryCreated { val falseBinary = if (new File("/bin/false").exists()) "/bin/false" else "/usr/bin/false" - val badStorage = new DiskStorage[IO](sConfig.copy(fixerCommand = Vector(falseBinary)), dConfig, cache) + val badStorage = + new DiskStorage[IO](sConfig.copy(fixerCommand = Vector(falseBinary)), contentTypeDetector, dConfig, cache) val file = "some/folder/my !file.txt" val absoluteFile = baseRootPath.resolve(Paths.get(file)) Files.createDirectories(absoluteFile.getParent) diff --git a/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/TarFlowSpec.scala b/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/TarFlowSpec.scala index 0a0f006519..7e06d5eb0d 100644 --- a/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/TarFlowSpec.scala +++ b/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/TarFlowSpec.scala @@ -2,7 +2,6 @@ package ch.epfl.bluebrain.nexus.storage import java.io.ByteArrayInputStream import java.nio.file.{Files, Path, Paths} - import akka.actor.ActorSystem import akka.stream.alpakka.file.scaladsl.Directory import akka.stream.scaladsl.{FileIO, Source} @@ -10,12 +9,12 @@ import akka.testkit.TestKit import akka.util.ByteString import ch.epfl.bluebrain.nexus.storage.utils.{EitherValues, IOEitherValues, Randomness} import org.apache.commons.compress.archivers.tar.TarArchiveInputStream -import org.apache.commons.io.FileUtils import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpecLike import org.scalatest.{BeforeAndAfterAll, Inspectors, OptionValues} import scala.annotation.tailrec +import scala.reflect.io.{Directory => ScalaDirectory} class TarFlowSpec extends TestKit(ActorSystem("TarFlowSpec")) @@ -34,7 +33,7 @@ class TarFlowSpec override def afterAll(): Unit = { super.afterAll() - FileUtils.cleanDirectory(basePath.toFile) + ScalaDirectory(basePath.toFile).deleteRecursively() () } diff --git a/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/attributes/AttributesComputationSpec.scala b/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/attributes/AttributesComputationSpec.scala index 1a43ea56bf..21196ae2c4 100644 --- a/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/attributes/AttributesComputationSpec.scala +++ b/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/attributes/AttributesComputationSpec.scala @@ -2,11 +2,11 @@ package ch.epfl.bluebrain.nexus.storage.attributes import java.nio.charset.StandardCharsets import java.nio.file.{Files, Paths} - import akka.actor.ActorSystem import akka.http.scaladsl.model.ContentTypes.`text/plain(UTF-8)` import akka.testkit.TestKit import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig import ch.epfl.bluebrain.nexus.storage.File.{Digest, FileAttributes} import ch.epfl.bluebrain.nexus.storage.StorageError.InternalError import ch.epfl.bluebrain.nexus.storage.utils.IOValues @@ -21,7 +21,8 @@ class AttributesComputationSpec with Matchers with IOValues { - implicit private val ec: ExecutionContextExecutor = system.dispatcher + implicit private val ec: ExecutionContextExecutor = system.dispatcher + implicit val contentTypeDetector: ContentTypeDetector = new ContentTypeDetector(MediaTypeDetectorConfig.Empty) private trait Ctx { val path = Files.createTempFile("storage-test", ".txt") diff --git a/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/attributes/ContentTypeDetectorSuite.scala b/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/attributes/ContentTypeDetectorSuite.scala new file mode 100644 index 0000000000..3bbd428070 --- /dev/null +++ b/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/attributes/ContentTypeDetectorSuite.scala @@ -0,0 +1,46 @@ +package ch.epfl.bluebrain.nexus.storage.attributes + +import akka.http.scaladsl.model.HttpCharsets.`UTF-8` +import akka.http.scaladsl.model.{ContentType, ContentTypes, MediaTypes} +import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig +import munit.FunSuite + +import java.nio.file.Paths + +class ContentTypeDetectorSuite extends FunSuite { + + private val jsonPath = Paths.get("content-type/file-example.json") + private val noExtensionPath = Paths.get("content-type/no-extension") + + test("Detect 'application/json' as content type") { + val detector = new ContentTypeDetector(MediaTypeDetectorConfig.Empty) + val expected = ContentTypes.`application/json` + assertEquals(detector(jsonPath, isDir = false), expected) + } + + test("Detect overridden content type") { + val customMediaType = MediaTypes.`application/vnd.api+json` + val detector = new ContentTypeDetector(MediaTypeDetectorConfig("json" -> MediaTypes.`application/vnd.api+json`)) + val expected = ContentType(customMediaType, () => `UTF-8`) + assertEquals(detector(jsonPath, isDir = false), expected) + } + + test("Detect overridden content type") { + val customMediaType = MediaTypes.`application/vnd.api+json` + val detector = new ContentTypeDetector(MediaTypeDetectorConfig("json" -> MediaTypes.`application/vnd.api+json`)) + val expected = ContentType(customMediaType, () => `UTF-8`) + assertEquals(detector(jsonPath, isDir = false), expected) + } + + test("Detect `application/octet-stream` as a default value") { + val detector = new ContentTypeDetector(MediaTypeDetectorConfig("json" -> MediaTypes.`application/vnd.api+json`)) + val expected = ContentTypes.`application/octet-stream` + assertEquals(detector(noExtensionPath, isDir = false), expected) + } + + test("Detect `application/x-tar` when the flag directory is set") { + val detector = new ContentTypeDetector(MediaTypeDetectorConfig("json" -> MediaTypes.`application/vnd.api+json`)) + val expected = ContentType(MediaTypes.`application/x-tar`, () => `UTF-8`) + assertEquals(detector(noExtensionPath, isDir = true), expected) + } +} diff --git a/tests/docker/config/delta-postgres.conf b/tests/docker/config/delta-postgres.conf index f142eaa3e7..3a51d05f9a 100644 --- a/tests/docker/config/delta-postgres.conf +++ b/tests/docker/config/delta-postgres.conf @@ -128,6 +128,14 @@ plugins { default-secret-key = "CHUTCHUT" } } + + files { + media-type-detector { + extensions { + custom = "application/custom" + } + } + } } project-deletion { diff --git a/tests/docker/config/storage.conf b/tests/docker/config/storage.conf new file mode 100644 index 0000000000..a4300e6137 --- /dev/null +++ b/tests/docker/config/storage.conf @@ -0,0 +1,34 @@ +app { + http { + interface = "0.0.0.0" + public-uri = "http://storage.dev.nise.bbp.epfl.ch" + } + + instance { + interface = "0.0.0.0" + } + + subject { + anonymous = false + realm = "internal" + name = "service-account-delta" + } + + storage { + root-volume = "/tmp" + protected-directory = "protected" + fixer-enabled = false + } + + delta { + public-iri = "https://test.nexus.bbp.epfl.ch" + internal-iri = "http://delta:8080" + } + + media-type-detector { + extensions { + custom = "application/custom" + } + } + +} \ No newline at end of file diff --git a/tests/docker/docker-compose.yml b/tests/docker/docker-compose.yml index 91e5828e8e..da5135697d 100644 --- a/tests/docker/docker-compose.yml +++ b/tests/docker/docker-compose.yml @@ -116,22 +116,16 @@ services: storage-service: container_name: "nexus-storage-service" image: bluebrain/nexus-storage:latest + environment: + STORAGE_CONFIG_FILE: "/config/storage.conf" + KAMON_ENABLED: "false" entrypoint: [ "./bin/storage", - "-Dapp.instance.interface=0.0.0.0", - "-Dapp.http.interface=0.0.0.0", - "-Dapp.http.public-uri=http://storage.tests.nexus.ocp.bbp.epfl.ch", - "-Dapp.subject.anonymous=false", - "-Dapp.subject.realm=internal", - "-Dapp.subject.name=service-account-delta", - "-Dapp.storage.root-volume=/tmp", - "-Dapp.storage.protected-directory=protected", - "-Dapp.storage.fixer-enabled=false", - "-Dapp.delta.public-iri=https://test.nexus.bbp.epfl.ch", - "-Dapp.delta.internal-iri=http://delta:8080", "-Dkamon.modules.prometheus-reporter.enabled=false", "-Dkamon.modules.jaeger.enabled=false" ] ports: - 8090:8090 + volumes: + - ./config:/config minio: image: minio/minio:RELEASE.2021-07-30T00-02-00Z diff --git a/tests/src/test/resources/kg/files/file.custom b/tests/src/test/resources/kg/files/file.custom new file mode 100644 index 0000000000..6b584e8ece --- /dev/null +++ b/tests/src/test/resources/kg/files/file.custom @@ -0,0 +1 @@ +content \ No newline at end of file diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/BaseSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/BaseSpec.scala index 8cbd1e1de2..324ce31b35 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/BaseSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/BaseSpec.scala @@ -239,6 +239,8 @@ trait BaseSpec private[tests] def expectCreated[A] = expect(StatusCodes.Created) + private[tests] def expectNotFound[A] = expect(StatusCodes.NotFound) + private[tests] def expectForbidden[A] = expect(StatusCodes.Forbidden) private[tests] def expectBadRequest[A] = expect(StatusCodes.BadRequest) diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala index 3d4d088693..1b3d009d8e 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala @@ -81,7 +81,7 @@ class HttpClient private (baseUrl: Uri, httpExt: HttpExt)(implicit as: ActorSyst ) } - def putAttachment[A]( + def uploadFile[A]( url: String, attachment: String, contentType: ContentType, diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala index da77ec16ba..4f8365b39a 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala @@ -37,11 +37,11 @@ class DiskStorageSpec extends StorageSpec { val payload2 = jsonContentOf("/kg/storages/disk-perms.json") for { - _ <- deltaClient.post[Json](s"/storages/$fullId", payload, Coyote) { (_, response) => + _ <- deltaClient.post[Json](s"/storages/$projectRef", payload, Coyote) { (_, response) => response.status shouldEqual StatusCodes.Created } - _ <- deltaClient.get[Json](s"/storages/$fullId/nxv:$storageId", Coyote) { (json, response) => - val expected = storageResponse(fullId, storageId, "resources/read", "files/write") + _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId", Coyote) { (json, response) => + val expected = storageResponse(projectRef, storageId, "resources/read", "files/write") filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) response.status shouldEqual StatusCodes.OK } @@ -49,12 +49,12 @@ class DiskStorageSpec extends StorageSpec { Permission(storageName, "read"), Permission(storageName, "write") ) - _ <- deltaClient.post[Json](s"/storages/$fullId", payload2, Coyote) { (_, response) => + _ <- deltaClient.post[Json](s"/storages/$projectRef", payload2, Coyote) { (_, response) => response.status shouldEqual StatusCodes.Created } storageId2 = s"${storageId}2" - _ <- deltaClient.get[Json](s"/storages/$fullId/nxv:$storageId2", Coyote) { (json, response) => - val expected = storageResponse(fullId, storageId2, s"$storageName/read", s"$storageName/write") + _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId2", Coyote) { (json, response) => + val expected = storageResponse(projectRef, storageId2, s"$storageName/read", s"$storageName/write") filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) response.status shouldEqual StatusCodes.OK } @@ -70,7 +70,7 @@ class DiskStorageSpec extends StorageSpec { "volume" -> Json.fromString(volume) ) - deltaClient.post[Json](s"/storages/$fullId", payload, Coyote) { (json, response) => + deltaClient.post[Json](s"/storages/$projectRef", payload, Coyote) { (json, response) => json shouldEqual jsonContentOf("/kg/storages/error.json", "volume" -> volume) response.status shouldEqual StatusCodes.BadRequest } @@ -85,7 +85,7 @@ class DiskStorageSpec extends StorageSpec { "mediaType" -> Json.fromString("image/png") ) - deltaClient.put[Json](s"/files/$fullId/linking.png", payload, Coyote) { (json, response) => + deltaClient.put[Json](s"/files/$projectRef/linking.png", payload, Coyote) { (json, response) => response.status shouldEqual StatusCodes.BadRequest json shouldEqual jsonContentOf("/kg/files/linking-notsupported.json", "org" -> orgId, "proj" -> projId) } diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/EventsSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/EventsSpec.scala index 7f57b90fc1..d673d2bde0 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/EventsSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/EventsSpec.scala @@ -122,7 +122,7 @@ class EventsSpec extends BaseSpec with Inspectors { response.status shouldEqual StatusCodes.OK } //FileCreated event - _ <- deltaClient.putAttachment[Json]( + _ <- deltaClient.uploadFile[Json]( s"/files/$id/attachment.json", contentOf("/kg/files/attachment.json"), ContentTypes.`application/json`, @@ -132,7 +132,7 @@ class EventsSpec extends BaseSpec with Inspectors { response.status shouldEqual StatusCodes.Created } //FileUpdated event - _ <- deltaClient.putAttachment[Json]( + _ <- deltaClient.uploadFile[Json]( s"/files/$id/attachment.json?rev=1", contentOf("/kg/files/attachment2.json"), ContentTypes.`application/json`, diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/MultiFetchSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/MultiFetchSpec.scala index ea23be932c..7ea95a3316 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/MultiFetchSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/MultiFetchSpec.scala @@ -32,7 +32,7 @@ class MultiFetchSpec extends BaseSpec { val createResources = for { // Creation _ <- deltaClient.put[Json](s"/resources/$ref11/_/nxv:resource", resourcePayload, Bob)(expectCreated) - _ <- deltaClient.putAttachment[Json]( + _ <- deltaClient.uploadFile[Json]( s"/files/$ref12/nxv:file", contentOf("/kg/files/attachment.json"), ContentTypes.`application/json`, diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/ProjectsDeletionSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/ProjectsDeletionSpec.scala index bbb7951696..e8767963cb 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/ProjectsDeletionSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/ProjectsDeletionSpec.scala @@ -118,14 +118,14 @@ final class ProjectsDeletionSpec extends BaseSpec with CirceEq with EitherValuab ref1 -> "https://bluebrain.github.io/nexus/vocabulary/defaultElasticSearchIndex", ref2 -> "https://bluebrain.github.io/nexus/vocabulary/defaultElasticSearchIndex" ) - _ <- deltaClient.putAttachment[Json]( + _ <- deltaClient.uploadFile[Json]( s"/files/$ref1/attachment.json", contentOf("/kg/files/attachment.json"), ContentTypes.`application/json`, "attachment.json", Bojack )(expectCreated) - _ <- deltaClient.putAttachment[Json]( + _ <- deltaClient.uploadFile[Json]( s"/files/$ref2/attachment.json", contentOf("/kg/files/attachment.json"), ContentTypes.`application/json`, diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala index d9f7d4019e..c1e8a22860 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala @@ -79,17 +79,17 @@ class RemoteStorageSpec extends StorageSpec { ) for { - _ <- deltaClient.post[Json](s"/storages/$fullId", payload, Coyote) { (json, response) => + _ <- deltaClient.post[Json](s"/storages/$projectRef", payload, Coyote) { (json, response) => if (response.status != StatusCodes.Created) { fail(s"Unexpected status '${response.status}', response:\n${json.spaces2}") } else succeed } - _ <- deltaClient.get[Json](s"/storages/$fullId/nxv:$storageId", Coyote) { (json, response) => - val expected = storageResponse(fullId, storageId, "resources/read", "files/write") + _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId", Coyote) { (json, response) => + val expected = storageResponse(projectRef, storageId, "resources/read", "files/write") filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) response.status shouldEqual StatusCodes.OK } - _ <- deltaClient.get[Json](s"/storages/$fullId/nxv:$storageId/source", Coyote) { (json, response) => + _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId/source", Coyote) { (json, response) => response.status shouldEqual StatusCodes.OK val expected = jsonContentOf( "/kg/storages/storage-source.json", @@ -103,12 +103,12 @@ class RemoteStorageSpec extends StorageSpec { Permission(storageName, "read"), Permission(storageName, "write") ) - _ <- deltaClient.post[Json](s"/storages/$fullId", payload2, Coyote) { (_, response) => + _ <- deltaClient.post[Json](s"/storages/$projectRef", payload2, Coyote) { (_, response) => response.status shouldEqual StatusCodes.Created } storageId2 = s"${storageId}2" - _ <- deltaClient.get[Json](s"/storages/$fullId/nxv:$storageId2", Coyote) { (json, response) => - val expected = storageResponse(fullId, storageId2, s"$storageName/read", s"$storageName/write") + _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId2", Coyote) { (json, response) => + val expected = storageResponse(projectRef, storageId2, s"$storageName/read", s"$storageName/write") filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) response.status shouldEqual StatusCodes.OK } @@ -116,8 +116,8 @@ class RemoteStorageSpec extends StorageSpec { } def putFile(name: String, content: String, storageId: String)(implicit position: Position) = { - deltaClient.putAttachment[Json]( - s"/files/$fullId/test-resource:$name?storage=nxv:${storageId}", + deltaClient.uploadFile[Json]( + s"/files/$projectRef/test-resource:$name?storage=nxv:${storageId}", content, MediaTypes.`text/plain`.toContentType(HttpCharsets.`UTF-8`), name, @@ -156,15 +156,17 @@ class RemoteStorageSpec extends StorageSpec { _ <- putFile("largefile13.txt", content, s"${storageId}2") _ <- putFile("largefile14.txt", content, s"${storageId}2") _ <- putFile("largefile15.txt", content, s"${storageId}2") - _ <- deltaClient.put[ByteString](s"/archives/$fullId/nxv:very-large-archive", payload, Coyote) { (_, response) => - before = System.currentTimeMillis() - response.status shouldEqual StatusCodes.Created - } _ <- - deltaClient.get[ByteString](s"/archives/$fullId/nxv:very-large-archive", Coyote, acceptAll) { (_, response) => - println(s"time taken to download archive: ${System.currentTimeMillis() - before}ms") - response.status shouldEqual StatusCodes.OK - contentType(response) shouldEqual MediaTypes.`application/x-tar`.toContentType + deltaClient.put[ByteString](s"/archives/$projectRef/nxv:very-large-archive", payload, Coyote) { (_, response) => + before = System.currentTimeMillis() + response.status shouldEqual StatusCodes.Created + } + _ <- + deltaClient.get[ByteString](s"/archives/$projectRef/nxv:very-large-archive", Coyote, acceptAll) { + (_, response) => + println(s"time taken to download archive: ${System.currentTimeMillis() - before}ms") + response.status shouldEqual StatusCodes.OK + contentType(response) shouldEqual MediaTypes.`application/x-tar`.toContentType } } yield { succeed @@ -182,7 +184,7 @@ class RemoteStorageSpec extends StorageSpec { "id" -> storageId ) - deltaClient.post[Json](s"/storages/$fullId", filterKey("folder")(payload), Coyote) { (_, response) => + deltaClient.post[Json](s"/storages/$projectRef", filterKey("folder")(payload), Coyote) { (_, response) => response.status shouldEqual StatusCodes.BadRequest } } @@ -198,44 +200,45 @@ class RemoteStorageSpec extends StorageSpec { "path" -> Json.fromString(s"file.txt"), "mediaType" -> Json.fromString("text/plain") ) - val fileId = s"${config.deltaUri}/resources/$fullId/_/file.txt" + val fileId = s"${config.deltaUri}/resources/$projectRef/_/file.txt" val expected = jsonContentOf( "/kg/files/remote-linked.json", replacements( Coyote, "id" -> fileId, - "self" -> fileSelf(fullId, fileId), + "self" -> fileSelf(projectRef, fileId), "filename" -> "file.txt", "storageId" -> s"${storageId}2", "storageType" -> storageType, - "projId" -> s"$fullId", - "project" -> s"${config.deltaUri}/projects/$fullId" + "projId" -> s"$projectRef", + "project" -> s"${config.deltaUri}/projects/$projectRef" ): _* ) - deltaClient.put[Json](s"/files/$fullId/file.txt?storage=nxv:${storageId}2", payload, Coyote) { (json, response) => - filterMetadataKeys.andThen(filterKey("_location"))(json) shouldEqual expected - response.status shouldEqual StatusCodes.Created + deltaClient.put[Json](s"/files/$projectRef/file.txt?storage=nxv:${storageId}2", payload, Coyote) { + (json, response) => + filterMetadataKeys.andThen(filterKey("_location"))(json) shouldEqual expected + response.status shouldEqual StatusCodes.Created } } "fetch eventually a linked file with updated attributes" in eventually { - val fileId = s"${config.deltaUri}/resources/$fullId/_/file.txt" + val fileId = s"${config.deltaUri}/resources/$projectRef/_/file.txt" val expected = jsonContentOf( "/kg/files/remote-updated-linked.json", replacements( Coyote, - "id" -> s"${config.deltaUri}/resources/$fullId/_/file.txt", - "self" -> fileSelf(fullId, fileId), + "id" -> s"${config.deltaUri}/resources/$projectRef/_/file.txt", + "self" -> fileSelf(projectRef, fileId), "filename" -> "file.txt", "storageId" -> s"${storageId}2", "storageType" -> storageType, - "projId" -> s"$fullId", - "project" -> s"${config.deltaUri}/projects/$fullId" + "projId" -> s"$projectRef", + "project" -> s"${config.deltaUri}/projects/$projectRef" ): _* ) - deltaClient.get[Json](s"/files/$fullId/file.txt", Coyote) { (json, response) => + deltaClient.get[Json](s"/files/$projectRef/file.txt", Coyote) { (json, response) => response.status shouldEqual StatusCodes.OK filterMetadataKeys.andThen(filterKey("_location"))(json) shouldEqual expected } @@ -248,7 +251,7 @@ class RemoteStorageSpec extends StorageSpec { "mediaType" -> Json.fromString("image/png") ) - deltaClient.put[Json](s"/files/$fullId/nonexistent.png?storage=nxv:${storageId}2", payload, Coyote) { + deltaClient.put[Json](s"/files/$projectRef/nonexistent.png?storage=nxv:${storageId}2", payload, Coyote) { (_, response) => response.status shouldEqual StatusCodes.BadRequest } @@ -289,8 +292,8 @@ class RemoteStorageSpec extends StorageSpec { deltaClient.get[Json]("/supervision/projections", Coyote) { (json1, _) => eventually { // update the file - deltaClient.putAttachment[Json]( - s"/files/$fullId/file.txt?storage=nxv:${storageId}2&rev=2", + deltaClient.uploadFile[Json]( + s"/files/$projectRef/file.txt?storage=nxv:${storageId}2&rev=2", contentOf("/kg/files/attachment.json"), ContentTypes.`application/json`, "file.txt", diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/S3StorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/S3StorageSpec.scala index 479a873e36..e71cc1d352 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/S3StorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/S3StorageSpec.scala @@ -102,21 +102,21 @@ class S3StorageSpec extends StorageSpec { ) for { - _ <- deltaClient.post[Json](s"/storages/$fullId", payload, Coyote) { (_, response) => + _ <- deltaClient.post[Json](s"/storages/$projectRef", payload, Coyote) { (_, response) => response.status shouldEqual StatusCodes.Created } - _ <- deltaClient.get[Json](s"/storages/$fullId/nxv:$storageId", Coyote) { (json, response) => - val expected = storageResponse(fullId, storageId, "resources/read", "files/write") + _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId", Coyote) { (json, response) => + val expected = storageResponse(projectRef, storageId, "resources/read", "files/write") filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) response.status shouldEqual StatusCodes.OK } _ <- permissionDsl.addPermissions(Permission(storageName, "read"), Permission(storageName, "write")) - _ <- deltaClient.post[Json](s"/storages/$fullId", payload2, Coyote) { (_, response) => + _ <- deltaClient.post[Json](s"/storages/$projectRef", payload2, Coyote) { (_, response) => response.status shouldEqual StatusCodes.Created } storageId2 = s"${storageId}2" - _ <- deltaClient.get[Json](s"/storages/$fullId/nxv:$storageId2", Coyote) { (json, response) => - val expected = storageResponse(fullId, storageId2, "s3/read", "s3/write") + _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId2", Coyote) { (json, response) => + val expected = storageResponse(projectRef, storageId2, "s3/read", "s3/write") .deepMerge(Json.obj("region" -> Json.fromString("eu-west-2"))) filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) response.status shouldEqual StatusCodes.OK @@ -133,7 +133,7 @@ class S3StorageSpec extends StorageSpec { "endpoint" -> s3Endpoint ) - deltaClient.post[Json](s"/storages/$fullId", payload, Coyote) { (json, response) => + deltaClient.post[Json](s"/storages/$projectRef", payload, Coyote) { (json, response) => json shouldEqual jsonContentOf("/kg/storages/s3-error.json") response.status shouldEqual StatusCodes.BadRequest } @@ -147,21 +147,22 @@ class S3StorageSpec extends StorageSpec { "path" -> Json.fromString(logoKey), "mediaType" -> Json.fromString("image/png") ) - val fileId = s"${config.deltaUri}/resources/$fullId/_/logo.png" - deltaClient.put[Json](s"/files/$fullId/logo.png?storage=nxv:${storageId}2", payload, Coyote) { (json, response) => - response.status shouldEqual StatusCodes.Created - filterMetadataKeys(json) shouldEqual - jsonContentOf( - "/kg/files/linking-metadata.json", - replacements( - Coyote, - "projId" -> fullId, - "self" -> fileSelf(fullId, fileId), - "endpoint" -> s3Endpoint, - "endpointBucket" -> s3BucketEndpoint, - "key" -> logoKey - ): _* - ) + val fileId = s"${config.deltaUri}/resources/$projectRef/_/logo.png" + deltaClient.put[Json](s"/files/$projectRef/logo.png?storage=nxv:${storageId}2", payload, Coyote) { + (json, response) => + response.status shouldEqual StatusCodes.Created + filterMetadataKeys(json) shouldEqual + jsonContentOf( + "/kg/files/linking-metadata.json", + replacements( + Coyote, + "projId" -> projectRef, + "self" -> fileSelf(projectRef, fileId), + "endpoint" -> s3Endpoint, + "endpointBucket" -> s3BucketEndpoint, + "key" -> logoKey + ): _* + ) } } } @@ -173,7 +174,7 @@ class S3StorageSpec extends StorageSpec { "mediaType" -> Json.fromString("image/png") ) - deltaClient.put[Json](s"/files/$fullId/nonexistent.png?storage=nxv:${storageId}2", payload, Coyote) { + deltaClient.put[Json](s"/files/$projectRef/nonexistent.png?storage=nxv:${storageId}2", payload, Coyote) { (json, response) => response.status shouldEqual StatusCodes.BadRequest json shouldEqual jsonContentOf( diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala index 680a1c7490..0cb3e84d54 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala @@ -1,7 +1,7 @@ package ch.epfl.bluebrain.nexus.tests.kg import akka.http.scaladsl.model.headers.{ContentDispositionTypes, HttpEncodings} -import akka.http.scaladsl.model.{ContentType, ContentTypes, HttpResponse, StatusCodes, Uri} +import akka.http.scaladsl.model._ import akka.util.ByteString import ch.epfl.bluebrain.nexus.testkit.CirceEq import ch.epfl.bluebrain.nexus.tests.BaseSpec @@ -11,7 +11,6 @@ import ch.epfl.bluebrain.nexus.tests.Optics._ import ch.epfl.bluebrain.nexus.tests.config.ConfigLoader._ import ch.epfl.bluebrain.nexus.tests.config.StorageConfig import ch.epfl.bluebrain.nexus.tests.iam.types.Permission -import ch.epfl.bluebrain.nexus.tests.iam.types.Permission.Organizations import com.typesafe.config.ConfigFactory import io.circe.Json import monix.bio.Task @@ -27,11 +26,11 @@ abstract class StorageSpec extends BaseSpec with CirceEq { val nxv = "https://bluebrain.github.io/nexus/vocabulary/" - private[tests] val orgId = genId() - private[tests] val projId = genId() - private[tests] val fullId = s"$orgId/$projId" + private[tests] val orgId = genId() + private[tests] val projId = genId() + private[tests] val projectRef = s"$orgId/$projId" - private[tests] val attachmentPrefix = s"${config.deltaUri}/resources/$fullId/_/" + private[tests] val attachmentPrefix = s"${config.deltaUri}/resources/$projectRef/_/" def storageName: String @@ -48,25 +47,11 @@ abstract class StorageSpec extends BaseSpec with CirceEq { uri.copy(path = uri.path / id).toString } - private[tests] val fileSelfPrefix = fileSelf(fullId, attachmentPrefix) - - "creating projects" should { - - "add necessary ACLs for user" in { - aclDsl.addPermission( - "/", - Coyote, - Organizations.Create - ) - } - - "succeed if payload is correct" in { - for { - _ <- adminDsl.createOrganization(orgId, orgId, Coyote) - _ <- adminDsl.createProject(orgId, projId, kgDsl.projectJson(name = fullId), Coyote) - } yield succeed - } + private[tests] val fileSelfPrefix = fileSelf(projectRef, attachmentPrefix) + override def beforeAll(): Unit = { + super.beforeAll() + createProjects(Coyote, orgId, projId).accepted } "creating a storage" should { @@ -76,7 +61,7 @@ abstract class StorageSpec extends BaseSpec with CirceEq { "wait for storages to be indexed" in { eventually { - deltaClient.get[Json](s"/storages/$fullId", Coyote) { (json, response) => + deltaClient.get[Json](s"/storages/$projectRef", Coyote) { (json, response) => response.status shouldEqual StatusCodes.OK _total.getOption(json).value shouldEqual 3 } @@ -87,127 +72,104 @@ abstract class StorageSpec extends BaseSpec with CirceEq { s"uploading an attachment against the $storageName storage" should { "upload empty file" in { - deltaClient.putAttachment[Json]( - s"/files/$fullId/empty?storage=nxv:$storageId", + deltaClient.uploadFile[Json]( + s"/files/$projectRef/empty?storage=nxv:$storageId", contentOf("/kg/files/empty"), ContentTypes.`text/plain(UTF-8)`, "empty", Coyote - ) { (_, response) => - response.status shouldEqual StatusCodes.Created - } + ) { expectCreated } } "fetch empty file" in { - deltaClient.get[ByteString](s"/files/$fullId/attachment:empty", Coyote, acceptAll) { (content, response) => - assertFetchAttachment( - response, + deltaClient.get[ByteString](s"/files/$projectRef/attachment:empty", Coyote, acceptAll) { + expectDownload( "empty", - ContentTypes.`text/plain(UTF-8)` + ContentTypes.`text/plain(UTF-8)`, + contentOf("/kg/files/empty") ) - content.utf8String shouldEqual contentOf("/kg/files/empty") } } "upload attachment with JSON" in { - deltaClient.putAttachment[Json]( - s"/files/$fullId/attachment.json?storage=nxv:$storageId", + deltaClient.uploadFile[Json]( + s"/files/$projectRef/attachment.json?storage=nxv:$storageId", contentOf("/kg/files/attachment.json"), - ContentTypes.`application/json`, + ContentTypes.NoContentType, "attachment.json", Coyote - ) { (_, response) => - response.status shouldEqual StatusCodes.Created - } + ) { expectCreated } } "fetch attachment" in { - deltaClient.get[ByteString](s"/files/$fullId/attachment:attachment.json", Coyote, acceptAll) { - (content, response) => - assertFetchAttachment( - response, - "attachment.json", - ContentTypes.`application/json` - ) - content.utf8String shouldEqual contentOf("/kg/files/attachment.json") + deltaClient.get[ByteString](s"/files/$projectRef/attachment:attachment.json", Coyote, acceptAll) { + expectDownload( + "attachment.json", + ContentTypes.`application/json`, + contentOf("/kg/files/attachment.json") + ) } } "fetch gzipped attachment" in { - deltaClient.get[ByteString](s"/files/$fullId/attachment:attachment.json", Coyote, gzipHeaders) { - (content, response) => - assertFetchAttachment( - response, - "attachment.json", - ContentTypes.`application/json` - ) - httpEncodings(response) shouldEqual Seq(HttpEncodings.gzip) - decodeGzip(content) shouldEqual contentOf("/kg/files/attachment.json") + deltaClient.get[ByteString](s"/files/$projectRef/attachment:attachment.json", Coyote, gzipHeaders) { + expectDownload( + "attachment.json", + ContentTypes.`application/json`, + contentOf("/kg/files/attachment.json"), + compressed = true + ) } } "update attachment with JSON" in { - deltaClient.putAttachment[Json]( - s"/files/$fullId/attachment.json?storage=nxv:$storageId&rev=1", + deltaClient.uploadFile[Json]( + s"/files/$projectRef/attachment.json?storage=nxv:$storageId&rev=1", contentOf("/kg/files/attachment2.json"), ContentTypes.`application/json`, "attachment.json", Coyote - ) { (_, response) => - response.status shouldEqual StatusCodes.OK - } + ) { expectOk } } "fetch updated attachment" in { - deltaClient.get[ByteString](s"/files/$fullId/attachment:attachment.json", Coyote, acceptAll) { - (content, response) => - assertFetchAttachment( - response, - "attachment.json", - ContentTypes.`application/json` - ) - content.utf8String shouldEqual contentOf("/kg/files/attachment2.json") + deltaClient.get[ByteString](s"/files/$projectRef/attachment:attachment.json", Coyote, acceptAll) { + expectDownload( + "attachment.json", + ContentTypes.`application/json`, + contentOf("/kg/files/attachment2.json") + ) } } "fetch previous revision of attachment" in { - deltaClient.get[ByteString](s"/files/$fullId/attachment:attachment.json?rev=1", Coyote, acceptAll) { - (content, response) => - assertFetchAttachment( - response, - "attachment.json", - ContentTypes.`application/json` - ) - content.utf8String shouldEqual contentOf("/kg/files/attachment.json") + deltaClient.get[ByteString](s"/files/$projectRef/attachment:attachment.json?rev=1", Coyote, acceptAll) { + expectDownload( + "attachment.json", + ContentTypes.`application/json`, + contentOf("/kg/files/attachment.json") + ) } } "upload second attachment to created storage" in { - deltaClient.putAttachment[Json]( - s"/files/$fullId/attachment2?storage=nxv:$storageId", + deltaClient.uploadFile[Json]( + s"/files/$projectRef/attachment2?storage=nxv:$storageId", contentOf("/kg/files/attachment2"), ContentTypes.NoContentType, "attachment2", Coyote - ) { (_, response) => - response.status shouldEqual StatusCodes.Created - } + ) { expectCreated } } "fetch second attachment" in { - deltaClient.get[ByteString](s"/files/$fullId/attachment:attachment2", Coyote, acceptAll) { (content, response) => - assertFetchAttachment( - response, - "attachment2" - ) - content.utf8String shouldEqual contentOf("/kg/files/attachment2") + deltaClient.get[ByteString](s"/files/$projectRef/attachment:attachment2", Coyote, acceptAll) { + expectDownload("attachment2", ContentTypes.`text/plain(UTF-8)`, contentOf("/kg/files/attachment2")) } } - "delete the attachment" in { - deltaClient.delete[Json](s"/files/$fullId/attachment:attachment.json?rev=2", Coyote) { (_, response) => - response.status shouldEqual StatusCodes.OK - } + "deprecate the attachment" in { + deltaClient.delete[Json](s"/files/$projectRef/attachment:attachment.json?rev=2", Coyote) { expectOk } } "fetch attachment metadata" in { @@ -217,16 +179,16 @@ abstract class StorageSpec extends BaseSpec with CirceEq { replacements( Coyote, "id" -> id, - "self" -> fileSelf(fullId, id), + "self" -> fileSelf(projectRef, id), "filename" -> "attachment.json", "storageId" -> storageId, "storageType" -> storageType, - "projId" -> s"$fullId", - "project" -> s"${config.deltaUri}/projects/$fullId" + "projId" -> s"$projectRef", + "project" -> s"${config.deltaUri}/projects/$projectRef" ): _* ) - deltaClient.get[Json](s"/files/$fullId/attachment:attachment.json", Coyote) { (json, response) => + deltaClient.get[Json](s"/files/$projectRef/attachment:attachment.json", Coyote) { (json, response) => response.status shouldEqual StatusCodes.OK locationPrefix.foreach { l => location.getOption(json).value should startWith(l) @@ -236,27 +198,23 @@ abstract class StorageSpec extends BaseSpec with CirceEq { } "attempt to upload a third attachment against an storage that does not exists" in { - deltaClient.putAttachment[Json]( - s"/files/$fullId/attachment3?storage=nxv:wrong-id", + deltaClient.uploadFile[Json]( + s"/files/$projectRef/attachment3?storage=nxv:wrong-id", contentOf("/kg/files/attachment2"), ContentTypes.NoContentType, "attachment2", Coyote - ) { (_, response) => - response.status shouldEqual StatusCodes.NotFound - } + ) { expectNotFound } } "fail to upload file against a storage with custom permissions" in { - deltaClient.putAttachment[Json]( - s"/files/$fullId/attachment3?storage=nxv:${storageId}2", + deltaClient.uploadFile[Json]( + s"/files/$projectRef/attachment3?storage=nxv:${storageId}2", contentOf("/kg/files/attachment2"), ContentTypes.NoContentType, "attachment2", Coyote - ) { (_, response) => - response.status shouldEqual StatusCodes.Forbidden - } + ) { expectForbidden } } "add ACLs for custom storage" in { @@ -268,36 +226,30 @@ abstract class StorageSpec extends BaseSpec with CirceEq { } "upload file against a storage with custom permissions" in { - deltaClient.putAttachment[Json]( - s"/files/$fullId/attachment3?storage=nxv:${storageId}2", + deltaClient.uploadFile[Json]( + s"/files/$projectRef/attachment3?storage=nxv:${storageId}2", contentOf("/kg/files/attachment2"), ContentTypes.NoContentType, "attachment2", Coyote - ) { (_, response) => - response.status shouldEqual StatusCodes.Created - } + ) { expectCreated } } } "deprecating a storage" should { "deprecate a storage" in { - deltaClient.delete[Json](s"/storages/$fullId/nxv:$storageId?rev=1", Coyote) { (_, response) => - response.status shouldEqual StatusCodes.OK - } + deltaClient.delete[Json](s"/storages/$projectRef/nxv:$storageId?rev=1", Coyote) { expectOk } } "reject uploading a new file against the deprecated storage" in { - deltaClient.putAttachment[Json]( - s"/files/$fullId/${genString()}?storage=nxv:$storageId", + deltaClient.uploadFile[Json]( + s"/files/$projectRef/${genString()}?storage=nxv:$storageId", "", ContentTypes.NoContentType, "attachment3", Coyote - ) { (_, response) => - response.status shouldEqual StatusCodes.BadRequest - } + ) { expectBadRequest } } "fetch second attachment metadata" in { @@ -308,15 +260,15 @@ abstract class StorageSpec extends BaseSpec with CirceEq { Coyote, "id" -> id, "storageId" -> storageId, - "self" -> fileSelf(fullId, id), + "self" -> fileSelf(projectRef, id), "storageType" -> storageType, - "projId" -> s"$fullId", - "project" -> s"${config.deltaUri}/projects/$fullId", + "projId" -> s"$projectRef", + "project" -> s"${config.deltaUri}/projects/$projectRef", "storageType" -> storageType ): _* ) - deltaClient.get[Json](s"/files/$fullId/attachment:attachment2", Coyote) { (json, response) => + deltaClient.get[Json](s"/files/$projectRef/attachment:attachment2", Coyote) { (json, response) => response.status shouldEqual StatusCodes.OK filterMetadataKeys.andThen(filterKey("_location"))(json) shouldEqual expected } @@ -325,30 +277,30 @@ abstract class StorageSpec extends BaseSpec with CirceEq { "getting statistics" should { "return the correct statistics" in eventually { - deltaClient.get[Json](s"/storages/$fullId/nxv:$storageId/statistics", Coyote) { (json, response) => + deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId/statistics", Coyote) { (json, response) => response.status shouldEqual StatusCodes.OK filterKey("lastProcessedEventDateTime")(json) shouldEqual jsonContentOf("/kg/storages/statistics.json") } } "fail for an unknown storage" in eventually { - deltaClient.get[Json](s"/storages/$fullId/nxv:fail/statistics", Coyote) { (json, response) => + deltaClient.get[Json](s"/storages/$projectRef/nxv:fail/statistics", Coyote) { (json, response) => response.status shouldEqual StatusCodes.NotFound json shouldEqual jsonContentOf( "/kg/storages/not-found.json", "storageId" -> (nxv + "fail"), - "projId" -> s"$fullId" + "projId" -> s"$projectRef" ) } } } "list files" in eventually { - deltaClient.get[Json](s"/files/$fullId", Coyote) { (json, response) => + deltaClient.get[Json](s"/files/$projectRef", Coyote) { (json, response) => response.status shouldEqual StatusCodes.OK val mapping = replacements( Coyote, - "project" -> fullId, + "project" -> projectRef, "fileSelfPrefix" -> fileSelfPrefix, "storageId" -> storageId, "storageType" -> storageType @@ -360,7 +312,7 @@ abstract class StorageSpec extends BaseSpec with CirceEq { } "query the default sparql view for files" in eventually { - val id = s"http://delta:8080/v1/resources/$fullId/_/attachment.json" + val id = s"http://delta:8080/v1/resources/$projectRef/_/attachment.json" val query = s""" |prefix nxv: @@ -405,18 +357,17 @@ abstract class StorageSpec extends BaseSpec with CirceEq { | """.stripMargin - deltaClient.sparqlQuery[Json](s"/views/$fullId/graph/sparql", query, Coyote) { (json, response) => + deltaClient.sparqlQuery[Json](s"/views/$projectRef/graph/sparql", query, Coyote) { (json, response) => response.status shouldEqual StatusCodes.OK val mapping = replacements( Coyote, - "project" -> fullId, - "self" -> fileSelf(fullId, id), + "project" -> projectRef, + "self" -> fileSelf(projectRef, id), "storageId" -> storageId ) val expected = jsonContentOf("/kg/files/sparql.json", mapping: _*) json should equalIgnoreArrayOrder(expected) } - } private def attachmentString(filename: String): String = { @@ -424,19 +375,21 @@ abstract class StorageSpec extends BaseSpec with CirceEq { s"=?UTF-8?B?$encodedFilename?=" } - private def assertFetchAttachment( - response: HttpResponse, + private def expectDownload( expectedFilename: String, - expectedContentType: ContentType - ): Assertion = { - assertFetchAttachment(response, expectedFilename) - contentType(response) shouldEqual expectedContentType - } - - private def assertFetchAttachment(response: HttpResponse, expectedFilename: String): Assertion = { - response.status shouldEqual StatusCodes.OK - dispositionType(response) shouldEqual ContentDispositionTypes.attachment - attachmentName(response) shouldEqual attachmentString(expectedFilename) - } - + expectedContentType: ContentType, + expectedContent: String, + compressed: Boolean = false + ) = + (content: ByteString, response: HttpResponse) => { + response.status shouldEqual StatusCodes.OK + dispositionType(response) shouldEqual ContentDispositionTypes.attachment + attachmentName(response) shouldEqual attachmentString(expectedFilename) + contentType(response) shouldEqual expectedContentType + if (compressed) { + httpEncodings(response) shouldEqual Seq(HttpEncodings.gzip) + decodeGzip(content) shouldEqual contentOf("/kg/files/attachment.json") + } else + content.utf8String shouldEqual expectedContent + } } From 5a86fcc0061b740b9fac47f7fc5087baf0deeaab Mon Sep 17 00:00:00 2001 From: Simon Dumas Date: Thu, 28 Sep 2023 12:07:37 +0200 Subject: [PATCH 2/9] Suppress TryGet warning --- .../nexus/delta/plugins/storage/files/FormDataExtractor.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala index 6787e50bf1..d9460df8a2 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala @@ -46,6 +46,7 @@ object FormDataExtractor { private val defaultContentType: ContentType.Binary = ContentTypes.`application/octet-stream` // Creating an unmarshaller defaulting to `application/octet-stream` as a content type + @SuppressWarnings(Array("TryGet")) implicit private val um: FromEntityUnmarshaller[Multipart.FormData] = MultipartUnmarshallers .multipartUnmarshaller[Multipart.FormData, Multipart.FormData.BodyPart, Multipart.FormData.BodyPart.Strict]( From a79b69fe759edbef6bfe79f4c2f82f903591aa31 Mon Sep 17 00:00:00 2001 From: Simon Dumas Date: Thu, 28 Sep 2023 17:37:07 +0200 Subject: [PATCH 3/9] Add integration tests --- .../storage/src/main/resources/storage.conf | 1 + .../storage/files/FormDataExtractor.scala | 69 +++-- .../storage/files/FormDataExtractorSpec.scala | 15 +- storage/src/main/resources/app.conf | 1 + .../test/resources/content-type}/file.custom | 0 .../kg/files/attachment-metadata.json | 4 +- .../test/resources/kg/files/attachment.json | 3 - tests/src/test/resources/kg/files/attachment2 | 1 - .../kg/files/attachment2-metadata.json | 6 +- .../test/resources/kg/files/attachment2.json | 3 - tests/src/test/resources/kg/files/empty | 0 tests/src/test/resources/kg/files/list.json | 16 +- .../resources/kg/files/remote-linked.json | 4 +- .../kg/files/remote-updated-linked.json | 2 +- tests/src/test/resources/kg/files/sparql.json | 4 +- .../resources/kg/storages/statistics.json | 5 - .../nexus/tests/kg/RemoteStorageSpec.scala | 198 +++++++++---- .../nexus/tests/kg/StorageSpec.scala | 269 +++++++++++------- 18 files changed, 381 insertions(+), 220 deletions(-) rename {tests/src/test/resources/kg/files => storage/src/test/resources/content-type}/file.custom (100%) delete mode 100644 tests/src/test/resources/kg/files/attachment.json delete mode 100644 tests/src/test/resources/kg/files/attachment2 delete mode 100644 tests/src/test/resources/kg/files/attachment2.json delete mode 100644 tests/src/test/resources/kg/files/empty delete mode 100644 tests/src/test/resources/kg/storages/statistics.json diff --git a/delta/plugins/storage/src/main/resources/storage.conf b/delta/plugins/storage/src/main/resources/storage.conf index ad5968955f..b927b43431 100644 --- a/delta/plugins/storage/src/main/resources/storage.conf +++ b/delta/plugins/storage/src/main/resources/storage.conf @@ -77,6 +77,7 @@ plugins.storage { # the files event log configuration event-log = ${app.defaults.event-log} + # Allows to define default media types for the given file extensions media-type-detector { extensions { #extension = "application/custom" diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala index d9460df8a2..74e57fae96 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala @@ -2,6 +2,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files import akka.actor.ActorSystem import akka.http.scaladsl.model.MediaTypes.`multipart/form-data` +import akka.http.scaladsl.model.Multipart.FormData import akka.http.scaladsl.model._ import akka.http.scaladsl.server._ import akka.http.scaladsl.unmarshalling.Unmarshaller.UnsupportedContentTypeException @@ -16,6 +17,8 @@ import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import monix.bio.IO import monix.execution.Scheduler +import scala.util.Try + sealed trait FormDataExtractor { /** @@ -71,33 +74,11 @@ object FormDataExtractor { ): IO[FileRejection, (FileDescription, BodyPartEntity)] = { val sizeLimit = Math.min(storageAvailableSpace.getOrElse(Long.MaxValue), maxFileSize) IO.deferFuture(um(entity.withSizeLimit(sizeLimit))) - .mapError { - case RejectionError(r) => - WrappedAkkaRejection(r) - case Unmarshaller.NoContentException => - WrappedAkkaRejection(RequestEntityExpectedRejection) - case x: UnsupportedContentTypeException => - WrappedAkkaRejection(UnsupportedRequestContentTypeRejection(x.supported, x.actualContentType)) - case x: IllegalArgumentException => - WrappedAkkaRejection(ValidationRejection(Option(x.getMessage).getOrElse(""), Some(x))) - case x: ExceptionWithErrorInfo => - WrappedAkkaRejection(MalformedRequestContentRejection(x.info.format(withDetail = false), x)) - case x => - WrappedAkkaRejection(MalformedRequestContentRejection(Option(x.getMessage).getOrElse(""), x)) - } + .mapError(onError) .flatMap { formData => IO.fromFuture( formData.parts - .mapAsync(parallelism = 1) { - case part if part.name == fieldName => - val filename = part.filename.getOrElse("file") - val contentType = detectContentType(filename, part.entity.contentType) - FileDescription(filename, contentType).runToFuture.map { desc => - Some(desc -> part.entity) - } - case part => - part.entity.discardBytes().future.as(None) - } + .mapAsync(parallelism = 1)(describe) .collect { case Some(values) => values } .toMat(Sink.headOption)(Keep.right) .run() @@ -110,15 +91,49 @@ object FormDataExtractor { } } + private def onError(th: Throwable) = th match { + case RejectionError(r) => + WrappedAkkaRejection(r) + case Unmarshaller.NoContentException => + WrappedAkkaRejection(RequestEntityExpectedRejection) + case x: UnsupportedContentTypeException => + WrappedAkkaRejection(UnsupportedRequestContentTypeRejection(x.supported, x.actualContentType)) + case x: IllegalArgumentException => + WrappedAkkaRejection(ValidationRejection(Option(x.getMessage).getOrElse(""), Some(x))) + case x: ExceptionWithErrorInfo => + WrappedAkkaRejection(MalformedRequestContentRejection(x.info.format(withDetail = false), x)) + case x => + WrappedAkkaRejection(MalformedRequestContentRejection(Option(x.getMessage).getOrElse(""), x)) + } + + private def describe(part: FormData.BodyPart) = part match { + case part if part.name == fieldName => + val filename = part.filename.getOrElse("file") + val contentType = detectContentType(filename, part.entity.contentType) + FileDescription(filename, contentType).runToFuture.map { desc => + Some(desc -> part.entity) + } + case part => + part.entity.discardBytes().future.as(None) + } + private def detectContentType(filename: String, contentTypeFromAkka: ContentType) = { val bodyDefinedContentType = Option.when(contentTypeFromAkka != defaultContentType)(contentTypeFromAkka) + val extensionOpt = FileUtils.extension(filename) + def detectFromConfig = for { - extension <- FileUtils.extension(filename) + extension <- extensionOpt customMediaType <- mediaTypeDetector.find(extension) - } yield ContentType(customMediaType, () => HttpCharsets.`UTF-8`) + } yield contentType(customMediaType) - bodyDefinedContentType.orElse(detectFromConfig).getOrElse(contentTypeFromAkka) + def detectAkkaFromExtension = extensionOpt.flatMap { e => + Try(MediaTypes.forExtension(e)).map(contentType).toOption + } + + bodyDefinedContentType.orElse(detectFromConfig).orElse(detectAkkaFromExtension).getOrElse(contentTypeFromAkka) } + + private def contentType(mediaType: MediaType) = ContentType(mediaType, () => HttpCharsets.`UTF-8`) } } diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractorSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractorSpec.scala index 5985d04e6d..eb83bac8e3 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractorSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractorSpec.scala @@ -48,15 +48,15 @@ class FormDataExtractorSpec .toEntity() "be extracted with the default content type" in { - val entity = createEntity("file", NoContentType, Some("file.txt")) + val entity = createEntity("file", NoContentType, Some("file")) - val expectedDescription = FileDescription(uuid, "file.txt", Some(`application/octet-stream`)) + val expectedDescription = FileDescription(uuid, "file", Some(`application/octet-stream`)) val (description, resultEntity) = extractor(iri, entity, 179, None).accepted description shouldEqual expectedDescription consume(resultEntity.dataBytes) shouldEqual content } - "be extracted with the custom media type" in { + "be extracted with the custom media type from the config" in { val entity = createEntity("file", NoContentType, Some("file.custom")) val expectedDescription = FileDescription(uuid, "file.custom", Some(customContentType)) val (description, resultEntity) = extractor(iri, entity, 2000, None).accepted @@ -64,6 +64,15 @@ class FormDataExtractorSpec consume(resultEntity.dataBytes) shouldEqual content } + "be extracted with the akka detection from the extension" in { + val entity = createEntity("file", NoContentType, Some("file.txt")) + + val expectedDescription = FileDescription(uuid, "file.txt", Some(`text/plain(UTF-8)`)) + val (description, resultEntity) = extractor(iri, entity, 179, None).accepted + description shouldEqual expectedDescription + consume(resultEntity.dataBytes) shouldEqual content + } + "be extracted with the provided content type header" in { val entity = createEntity("file", `text/plain(UTF-8)`, Some("file.custom")) val expectedDescription = FileDescription(uuid, "file.custom", Some(`text/plain(UTF-8)`)) diff --git a/storage/src/main/resources/app.conf b/storage/src/main/resources/app.conf index db55274922..27ff92a2db 100644 --- a/storage/src/main/resources/app.conf +++ b/storage/src/main/resources/app.conf @@ -43,6 +43,7 @@ app { fixer-command = [] } + # Allows to define default media types for the given file extensions media-type-detector { extensions { #extension = "application/custom" diff --git a/tests/src/test/resources/kg/files/file.custom b/storage/src/test/resources/content-type/file.custom similarity index 100% rename from tests/src/test/resources/kg/files/file.custom rename to storage/src/test/resources/content-type/file.custom diff --git a/tests/src/test/resources/kg/files/attachment-metadata.json b/tests/src/test/resources/kg/files/attachment-metadata.json index ffc94418f7..a94cbb6171 100644 --- a/tests/src/test/resources/kg/files/attachment-metadata.json +++ b/tests/src/test/resources/kg/files/attachment-metadata.json @@ -10,10 +10,10 @@ "@type" : "{{storageType}}", "_rev": 1 }, - "_bytes": 52, + "_bytes": 42, "_digest": { "_algorithm": "SHA-256", - "_value": "0e5ce4e33df07eb312e462d9b4646aebf44e7c4d758891e632dbb0533df85e3f" + "_value": "9144c8a8c435aff3262ba0a4620590f774c1d86a13956619723052eca6546c64" }, "_filename": "{{filename}}", "_mediaType": "application/json", diff --git a/tests/src/test/resources/kg/files/attachment.json b/tests/src/test/resources/kg/files/attachment.json deleted file mode 100644 index 149d45ade7..0000000000 --- a/tests/src/test/resources/kg/files/attachment.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "this": ["is", "a", "test", "attachment"] -} \ No newline at end of file diff --git a/tests/src/test/resources/kg/files/attachment2 b/tests/src/test/resources/kg/files/attachment2 deleted file mode 100644 index 11644f85bb..0000000000 --- a/tests/src/test/resources/kg/files/attachment2 +++ /dev/null @@ -1 +0,0 @@ -this is another attachment \ No newline at end of file diff --git a/tests/src/test/resources/kg/files/attachment2-metadata.json b/tests/src/test/resources/kg/files/attachment2-metadata.json index 383b92cd84..ebf44c7947 100644 --- a/tests/src/test/resources/kg/files/attachment2-metadata.json +++ b/tests/src/test/resources/kg/files/attachment2-metadata.json @@ -10,13 +10,13 @@ "@type" : "{{storageType}}", "_rev": 1 }, - "_bytes": 26, + "_bytes": 9, "_digest": { "_algorithm": "SHA-256", - "_value": "62ad07ee5e775552034186daafd7d77054620f2bc9fb8da629fd7af24b4c84e7" + "_value": "2b67fbb2c302811c3e4941183b1b2037774ee7dfc8b633eef1afa2231d001615" }, "_filename": "attachment2", - "_mediaType": "text/plain; charset=UTF-8", + "_mediaType": "application/octet-stream", "_origin" : "Client", "_incoming": "{{self}}/incoming", "_outgoing": "{{self}}/outgoing", diff --git a/tests/src/test/resources/kg/files/attachment2.json b/tests/src/test/resources/kg/files/attachment2.json deleted file mode 100644 index 5ff66791fa..0000000000 --- a/tests/src/test/resources/kg/files/attachment2.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "this": ["is", "a", "test", "attachment", "2"] -} \ No newline at end of file diff --git a/tests/src/test/resources/kg/files/empty b/tests/src/test/resources/kg/files/empty deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/src/test/resources/kg/files/list.json b/tests/src/test/resources/kg/files/list.json index 38f634a71c..2c7a37df17 100644 --- a/tests/src/test/resources/kg/files/list.json +++ b/tests/src/test/resources/kg/files/list.json @@ -8,13 +8,13 @@ { "@id": "{{deltaUri}}/resources/{{project}}/_/attachment.json", "@type": "File", - "_bytes": 52, + "_bytes": 42, "_constrainedBy": "https://bluebrain.github.io/nexus/schemas/files.json", "_createdBy": "{{deltaUri}}/realms/{{realm}}/users/{{user}}", "_deprecated": true, "_digest": { "_algorithm": "SHA-256", - "_value": "0e5ce4e33df07eb312e462d9b4646aebf44e7c4d758891e632dbb0533df85e3f" + "_value": "9144c8a8c435aff3262ba0a4620590f774c1d86a13956619723052eca6546c64" }, "_filename": "attachment.json", "_incoming": "{{fileSelfPrefix}}attachment.json/incoming", @@ -58,17 +58,17 @@ { "@id": "{{deltaUri}}/resources/{{project}}/_/attachment2", "@type": "File", - "_bytes": 26, + "_bytes": 9, "_constrainedBy": "https://bluebrain.github.io/nexus/schemas/files.json", "_createdBy": "{{deltaUri}}/realms/{{realm}}/users/{{user}}", "_deprecated": false, "_digest": { "_algorithm": "SHA-256", - "_value": "62ad07ee5e775552034186daafd7d77054620f2bc9fb8da629fd7af24b4c84e7" + "_value": "2b67fbb2c302811c3e4941183b1b2037774ee7dfc8b633eef1afa2231d001615" }, "_filename": "attachment2", "_incoming": "{{fileSelfPrefix}}attachment2/incoming", - "_mediaType": "text/plain; charset=UTF-8", + "_mediaType": "application/octet-stream", "_outgoing": "{{fileSelfPrefix}}attachment2/outgoing", "_project": "{{deltaUri}}/projects/{{project}}", "_rev": 1, @@ -83,17 +83,17 @@ { "@id": "{{deltaUri}}/resources/{{project}}/_/attachment3", "@type": "File", - "_bytes": 26, + "_bytes": 9, "_constrainedBy": "https://bluebrain.github.io/nexus/schemas/files.json", "_createdBy": "{{deltaUri}}/realms/{{realm}}/users/{{user}}", "_deprecated": false, "_digest": { "_algorithm": "SHA-256", - "_value": "62ad07ee5e775552034186daafd7d77054620f2bc9fb8da629fd7af24b4c84e7" + "_value": "2b67fbb2c302811c3e4941183b1b2037774ee7dfc8b633eef1afa2231d001615" }, "_filename": "attachment2", "_incoming": "{{fileSelfPrefix}}attachment3/incoming", - "_mediaType": "text/plain; charset=UTF-8", + "_mediaType": "application/octet-stream", "_outgoing": "{{fileSelfPrefix}}attachment3/outgoing", "_project": "{{deltaUri}}/projects/{{project}}", "_rev": 1, diff --git a/tests/src/test/resources/kg/files/remote-linked.json b/tests/src/test/resources/kg/files/remote-linked.json index 0ea6dd24fa..a9a1c64a25 100644 --- a/tests/src/test/resources/kg/files/remote-linked.json +++ b/tests/src/test/resources/kg/files/remote-linked.json @@ -13,7 +13,9 @@ "_value": "" }, "_filename": "{{filename}}", - "_mediaType": "text/plain", + {{#if mediaType}} + "_mediaType": "{{mediaType}}", + {{/if}} "_origin": "Storage", "_incoming": "{{self}}/incoming", "_outgoing": "{{self}}/outgoing", diff --git a/tests/src/test/resources/kg/files/remote-updated-linked.json b/tests/src/test/resources/kg/files/remote-updated-linked.json index ad0aadccd5..2c3e428a31 100644 --- a/tests/src/test/resources/kg/files/remote-updated-linked.json +++ b/tests/src/test/resources/kg/files/remote-updated-linked.json @@ -15,7 +15,7 @@ }, "_filename": "{{filename}}", "_incoming": "{{self}}/incoming", - "_mediaType": "text/plain", + "_mediaType": "{{mediaType}}", "_origin": "Storage", "_outgoing": "{{self}}/outgoing", "_project": "{{project}}", diff --git a/tests/src/test/resources/kg/files/sparql.json b/tests/src/test/resources/kg/files/sparql.json index c9ae92c7f8..d84c92092d 100644 --- a/tests/src/test/resources/kg/files/sparql.json +++ b/tests/src/test/resources/kg/files/sparql.json @@ -41,7 +41,7 @@ "object": { "datatype": "http://www.w3.org/2001/XMLSchema#integer", "type": "literal", - "value": "52" + "value": "42" }, "predicate": { "type": "uri", @@ -55,7 +55,7 @@ { "object": { "type": "literal", - "value": "0e5ce4e33df07eb312e462d9b4646aebf44e7c4d758891e632dbb0533df85e3f" + "value": "9144c8a8c435aff3262ba0a4620590f774c1d86a13956619723052eca6546c64" }, "predicate": { "type": "uri", diff --git a/tests/src/test/resources/kg/storages/statistics.json b/tests/src/test/resources/kg/storages/statistics.json deleted file mode 100644 index ecc1049f3d..0000000000 --- a/tests/src/test/resources/kg/storages/statistics.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "@context": "https://bluebrain.github.io/nexus/contexts/storages.json", - "files": 4, - "spaceUsed": 125 -} \ No newline at end of file diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala index c1e8a22860..cef8374967 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala @@ -8,6 +8,7 @@ import ch.epfl.bluebrain.nexus.tests.Optics.{filterKey, filterMetadataKeys, proj import ch.epfl.bluebrain.nexus.tests.iam.types.Permission import ch.epfl.bluebrain.nexus.tests.iam.types.Permission.Supervision import io.circe.generic.semiauto.deriveDecoder +import io.circe.syntax.KeyOps import io.circe.{Decoder, Json} import monix.bio.Task import org.scalactic.source.Position @@ -173,7 +174,7 @@ class RemoteStorageSpec extends StorageSpec { } } - "creating a remote storage" should { + "Creating a remote storage" should { "fail creating a RemoteDiskStorage without folder" in { val payload = jsonContentOf( "/kg/storages/remote-disk.json", @@ -190,72 +191,165 @@ class RemoteStorageSpec extends StorageSpec { } } - s"Linking in Remote storage" should { - "link an existing file" in { - val createFile = s"echo 'file content' > /tmp/$remoteFolder/file.txt" - s"docker exec nexus-storage-service bash -c \"$createFile\"".! + def createFile(filename: String) = Task.delay { + val createFile = s"echo 'file content' > /tmp/$remoteFolder/$filename" + s"docker exec nexus-storage-service bash -c \"$createFile\"".! + } - val payload = Json.obj( - "filename" -> Json.fromString("file.txt"), - "path" -> Json.fromString(s"file.txt"), - "mediaType" -> Json.fromString("text/plain") - ) - val fileId = s"${config.deltaUri}/resources/$projectRef/_/file.txt" - val expected = jsonContentOf( - "/kg/files/remote-linked.json", - replacements( - Coyote, - "id" -> fileId, - "self" -> fileSelf(projectRef, fileId), - "filename" -> "file.txt", - "storageId" -> s"${storageId}2", - "storageType" -> storageType, - "projId" -> s"$projectRef", - "project" -> s"${config.deltaUri}/projects/$projectRef" - ): _* - ) + def linkPayload(filename: String, path: String, mediaType: Option[String]) = + Json.obj( + "filename" := filename, + "path" := path, + "mediaType" := mediaType + ) - deltaClient.put[Json](s"/files/$projectRef/file.txt?storage=nxv:${storageId}2", payload, Coyote) { - (json, response) => - filterMetadataKeys.andThen(filterKey("_location"))(json) shouldEqual expected - response.status shouldEqual StatusCodes.Created - } + def linkFile(payload: Json)(fileId: String, filename: String, mediaType: Option[String]) = { + val expected = jsonContentOf( + "/kg/files/remote-linked.json", + replacements( + Coyote, + "id" -> fileId, + "self" -> fileSelf(projectRef, fileId), + "filename" -> filename, + "mediaType" -> mediaType.orNull, + "storageId" -> s"${storageId}2", + "storageType" -> storageType, + "projId" -> s"$projectRef", + "project" -> s"${config.deltaUri}/projects/$projectRef" + ): _* + ) + deltaClient.put[Json](s"/files/$projectRef/$filename?storage=nxv:${storageId}2", payload, Coyote) { + (json, response) => + filterMetadataKeys.andThen(filterKey("_location"))(json) shouldEqual expected + response.status shouldEqual StatusCodes.Created } + } - "fetch eventually a linked file with updated attributes" in eventually { - val fileId = s"${config.deltaUri}/resources/$projectRef/_/file.txt" - val expected = jsonContentOf( - "/kg/files/remote-updated-linked.json", - replacements( - Coyote, - "id" -> s"${config.deltaUri}/resources/$projectRef/_/file.txt", - "self" -> fileSelf(projectRef, fileId), - "filename" -> "file.txt", - "storageId" -> s"${storageId}2", - "storageType" -> storageType, - "projId" -> s"$projectRef", - "project" -> s"${config.deltaUri}/projects/$projectRef" - ): _* - ) + def fetchUpdatedLinkedFile(fileId: String, filename: String, mediaType: String) = { + val expected = jsonContentOf( + "/kg/files/remote-updated-linked.json", + replacements( + Coyote, + "id" -> fileId, + "self" -> fileSelf(projectRef, fileId), + "filename" -> filename, + "mediaType" -> mediaType, + "storageId" -> s"${storageId}2", + "storageType" -> storageType, + "projId" -> s"$projectRef", + "project" -> s"${config.deltaUri}/projects/$projectRef" + ): _* + ) - deltaClient.get[Json](s"/files/$projectRef/file.txt", Coyote) { (json, response) => + eventually { + deltaClient.get[Json](s"/files/$projectRef/$filename", Coyote) { (json, response) => response.status shouldEqual StatusCodes.OK filterMetadataKeys.andThen(filterKey("_location"))(json) shouldEqual expected } } + } - "fail to link a nonexistent file" in { - val payload = Json.obj( - "filename" -> Json.fromString("logo.png"), - "path" -> Json.fromString("non/existent.png"), - "mediaType" -> Json.fromString("image/png") - ) + "Linking a custom file providing a media type for a .custom file" should { + + val filename = "link_file.custom" + val fileId = s"${config.deltaUri}/resources/$projectRef/_/$filename" + + "succeed" in { + + val mediaType = "application/json" + val payload = linkPayload(filename, filename, Some(mediaType)) + + for { + _ <- createFile(filename) + // Get a first response without the digest + _ <- linkFile(payload)(fileId, filename, Some(mediaType)) + // Eventually + } yield succeed + } + + "fetch eventually a linked file with updated attributes" in { + val mediaType = "application/json" + fetchUpdatedLinkedFile(fileId, filename, mediaType) + } + } + + "Linking a file without a media type for a .custom file" should { + + val filename = "link_file_no_media_type.custom" + val fileId = s"${config.deltaUri}/resources/$projectRef/_/$filename" + + "succeed" in { + val payload = linkPayload(filename, filename, None) + + for { + _ <- createFile(filename) + // Get a first response without the digest + _ <- linkFile(payload)(fileId, filename, None) + // Eventually + } yield succeed + } + + "fetch eventually a linked file with updated attributes detecting application/custom from config" in { + val mediaType = "application/custom" + fetchUpdatedLinkedFile(fileId, filename, mediaType) + } + } + + "Linking a file without a media type for a .txt file" should { + + val filename = "link_file.txt" + val fileId = s"${config.deltaUri}/resources/$projectRef/_/$filename" + + "succeed" in { + val payload = linkPayload(filename, filename, None) + + for { + _ <- createFile(filename) + // Get a first response without the digest + _ <- linkFile(payload)(fileId, filename, None) + // Eventually + } yield succeed + } + + "fetch eventually a linked file with updated attributes detecting text/plain from akka" in { + val mediaType = "text/plain; charset=UTF-8" + fetchUpdatedLinkedFile(fileId, filename, mediaType) + } + } + + "Linking a file without a media type for a file without extension" should { + + val filename = "link_file_no_extension" + val fileId = s"${config.deltaUri}/resources/$projectRef/_/$filename" + + "succeed" in { + val payload = linkPayload(filename, filename, None) + + for { + _ <- createFile(filename) + // Get a first response without the digest + _ <- linkFile(payload)(fileId, filename, None) + // Eventually + } yield succeed + } + + "fetch eventually a linked file with updated attributes falling back to default mediaType" in { + val mediaType = "application/octet-stream" + fetchUpdatedLinkedFile(fileId, filename, mediaType) + } + } + + "Linking providing a nonexistent file" should { + + "fail" in { + val payload = linkPayload("logo.png", "non/existent.png", Some("image/png")) deltaClient.put[Json](s"/files/$projectRef/nonexistent.png?storage=nxv:${storageId}2", payload, Coyote) { (_, response) => response.status shouldEqual StatusCodes.BadRequest } } + } "The file-attributes-updated projection description" should { @@ -294,7 +388,7 @@ class RemoteStorageSpec extends StorageSpec { // update the file deltaClient.uploadFile[Json]( s"/files/$projectRef/file.txt?storage=nxv:${storageId}2&rev=2", - contentOf("/kg/files/attachment.json"), + s"""{ "json": "content"}""", ContentTypes.`application/json`, "file.txt", Coyote diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala index 0cb3e84d54..656ebdb761 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala @@ -13,6 +13,7 @@ import ch.epfl.bluebrain.nexus.tests.config.StorageConfig import ch.epfl.bluebrain.nexus.tests.iam.types.Permission import com.typesafe.config.ConfigFactory import io.circe.Json +import io.circe.optics.JsonPath.root import monix.bio.Task import monix.execution.Scheduler.Implicits.global import org.apache.commons.codec.Charsets @@ -54,8 +55,8 @@ abstract class StorageSpec extends BaseSpec with CirceEq { createProjects(Coyote, orgId, projId).accepted } - "creating a storage" should { - s"succeed creating a $storageName storage" in { + "Creating a storage" should { + s"succeed for a $storageName storage" in { createStorages } @@ -69,110 +70,95 @@ abstract class StorageSpec extends BaseSpec with CirceEq { } } - s"uploading an attachment against the $storageName storage" should { + "An empty file" should { - "upload empty file" in { + val emptyFileContent = "" + + "be successfully uploaded" in { deltaClient.uploadFile[Json]( s"/files/$projectRef/empty?storage=nxv:$storageId", - contentOf("/kg/files/empty"), + emptyFileContent, ContentTypes.`text/plain(UTF-8)`, "empty", Coyote ) { expectCreated } } - "fetch empty file" in { + "be downloaded" in { deltaClient.get[ByteString](s"/files/$projectRef/attachment:empty", Coyote, acceptAll) { - expectDownload( - "empty", - ContentTypes.`text/plain(UTF-8)`, - contentOf("/kg/files/empty") - ) + expectDownload("empty", ContentTypes.`text/plain(UTF-8)`, emptyFileContent) } } + } + + "A json file" should { - "upload attachment with JSON" in { + val jsonFileContent = """{ "initial": ["is", "a", "test", "file"] }""" + val updatedJsonFileContent = """{ "updated": ["is", "a", "test", "file"] }""" + + "be uploaded" in { deltaClient.uploadFile[Json]( s"/files/$projectRef/attachment.json?storage=nxv:$storageId", - contentOf("/kg/files/attachment.json"), + jsonFileContent, ContentTypes.NoContentType, "attachment.json", Coyote - ) { expectCreated } + ) { + expectCreated + } } - "fetch attachment" in { + "be downloaded" in { deltaClient.get[ByteString](s"/files/$projectRef/attachment:attachment.json", Coyote, acceptAll) { - expectDownload( - "attachment.json", - ContentTypes.`application/json`, - contentOf("/kg/files/attachment.json") - ) + expectDownload("attachment.json", ContentTypes.`application/json`, jsonFileContent) } } - "fetch gzipped attachment" in { + "be downloaded as gzip" in { deltaClient.get[ByteString](s"/files/$projectRef/attachment:attachment.json", Coyote, gzipHeaders) { - expectDownload( - "attachment.json", - ContentTypes.`application/json`, - contentOf("/kg/files/attachment.json"), - compressed = true - ) + expectDownload("attachment.json", ContentTypes.`application/json`, jsonFileContent, compressed = true) } } - "update attachment with JSON" in { + "be updated" in { deltaClient.uploadFile[Json]( s"/files/$projectRef/attachment.json?storage=nxv:$storageId&rev=1", - contentOf("/kg/files/attachment2.json"), + updatedJsonFileContent, ContentTypes.`application/json`, "attachment.json", Coyote - ) { expectOk } + ) { + expectOk + } } - "fetch updated attachment" in { + "download the updated file" in { deltaClient.get[ByteString](s"/files/$projectRef/attachment:attachment.json", Coyote, acceptAll) { expectDownload( "attachment.json", ContentTypes.`application/json`, - contentOf("/kg/files/attachment2.json") + updatedJsonFileContent ) } } - "fetch previous revision of attachment" in { + "download the previous revision" in { deltaClient.get[ByteString](s"/files/$projectRef/attachment:attachment.json?rev=1", Coyote, acceptAll) { expectDownload( "attachment.json", ContentTypes.`application/json`, - contentOf("/kg/files/attachment.json") + jsonFileContent ) } } - "upload second attachment to created storage" in { - deltaClient.uploadFile[Json]( - s"/files/$projectRef/attachment2?storage=nxv:$storageId", - contentOf("/kg/files/attachment2"), - ContentTypes.NoContentType, - "attachment2", - Coyote - ) { expectCreated } - } - - "fetch second attachment" in { - deltaClient.get[ByteString](s"/files/$projectRef/attachment:attachment2", Coyote, acceptAll) { - expectDownload("attachment2", ContentTypes.`text/plain(UTF-8)`, contentOf("/kg/files/attachment2")) + "deprecate the file" in { + deltaClient.delete[Json](s"/files/$projectRef/attachment:attachment.json?rev=2", Coyote) { + expectOk } } - "deprecate the attachment" in { - deltaClient.delete[Json](s"/files/$projectRef/attachment:attachment.json?rev=2", Coyote) { expectOk } - } - - "fetch attachment metadata" in { + "have the expected metadata" in { val id = s"${attachmentPrefix}attachment.json" val expected = jsonContentOf( "/kg/files/attachment-metadata.json", @@ -196,90 +182,79 @@ abstract class StorageSpec extends BaseSpec with CirceEq { filterMetadataKeys.andThen(filterKey("_location"))(json) shouldEqual expected } } + } - "attempt to upload a third attachment against an storage that does not exists" in { - deltaClient.uploadFile[Json]( - s"/files/$projectRef/attachment3?storage=nxv:wrong-id", - contentOf("/kg/files/attachment2"), - ContentTypes.NoContentType, - "attachment2", - Coyote - ) { expectNotFound } - } + "A file without extension" should { - "fail to upload file against a storage with custom permissions" in { + val textFileContent = "text file" + + "be uploaded" in { deltaClient.uploadFile[Json]( - s"/files/$projectRef/attachment3?storage=nxv:${storageId}2", - contentOf("/kg/files/attachment2"), + s"/files/$projectRef/attachment2?storage=nxv:$storageId", + textFileContent, ContentTypes.NoContentType, "attachment2", Coyote - ) { expectForbidden } + ) { + expectCreated + } } - "add ACLs for custom storage" in { - aclDsl.addPermissions( - "/", - Coyote, - Set(Permission(storageName, "read"), Permission(storageName, "write")) - ) + "be downloaded" in { + deltaClient.get[ByteString](s"/files/$projectRef/attachment:attachment2", Coyote, acceptAll) { + expectDownload("attachment2", ContentTypes.`application/octet-stream`, textFileContent) + } } + } + + "Uploading a file against a unknown storage" should { - "upload file against a storage with custom permissions" in { + val textFileContent = "text file" + + "fail" in { deltaClient.uploadFile[Json]( - s"/files/$projectRef/attachment3?storage=nxv:${storageId}2", - contentOf("/kg/files/attachment2"), + s"/files/$projectRef/attachment3?storage=nxv:wrong-id", + textFileContent, ContentTypes.NoContentType, "attachment2", Coyote - ) { expectCreated } + ) { expectNotFound } } } - "deprecating a storage" should { + "Uploading a file against a storage with custom permissions" should { - "deprecate a storage" in { - deltaClient.delete[Json](s"/storages/$projectRef/nxv:$storageId?rev=1", Coyote) { expectOk } - } + val textFileContent = "text file" - "reject uploading a new file against the deprecated storage" in { + def uploadStorageWithCustomPermissions: ((Json, HttpResponse) => Assertion) => Task[Assertion] = deltaClient.uploadFile[Json]( - s"/files/$projectRef/${genString()}?storage=nxv:$storageId", - "", + s"/files/$projectRef/attachment3?storage=nxv:${storageId}2", + textFileContent, ContentTypes.NoContentType, - "attachment3", + "attachment2", Coyote - ) { expectBadRequest } - } - - "fetch second attachment metadata" in { - val id = s"${attachmentPrefix}attachment2" - val expected = jsonContentOf( - "/kg/files/attachment2-metadata.json", - replacements( - Coyote, - "id" -> id, - "storageId" -> storageId, - "self" -> fileSelf(projectRef, id), - "storageType" -> storageType, - "projId" -> s"$projectRef", - "project" -> s"${config.deltaUri}/projects/$projectRef", - "storageType" -> storageType - ): _* ) - deltaClient.get[Json](s"/files/$projectRef/attachment:attachment2", Coyote) { (json, response) => - response.status shouldEqual StatusCodes.OK - filterMetadataKeys.andThen(filterKey("_location"))(json) shouldEqual expected - } + "fail without these custom permissions" in { + uploadStorageWithCustomPermissions { expectForbidden } + } + + "succeed with the appropriate permissions" in { + val permissions = Set(Permission(storageName, "read"), Permission(storageName, "write")) + for { + _ <- aclDsl.addPermissions("/", Coyote, permissions) + _ <- uploadStorageWithCustomPermissions { expectCreated } + } yield (succeed) } } - "getting statistics" should { + "Getting statistics" should { "return the correct statistics" in eventually { deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId/statistics", Coyote) { (json, response) => response.status shouldEqual StatusCodes.OK - filterKey("lastProcessedEventDateTime")(json) shouldEqual jsonContentOf("/kg/storages/statistics.json") + val expected = + json"""{ "@context": "https://bluebrain.github.io/nexus/contexts/storages.json", "files": 4, "spaceUsed": 93 }""" + filterKey("lastProcessedEventDateTime")(json) shouldEqual expected } } @@ -295,7 +270,7 @@ abstract class StorageSpec extends BaseSpec with CirceEq { } } - "list files" in eventually { + "List files" in eventually { deltaClient.get[Json](s"/files/$projectRef", Coyote) { (json, response) => response.status shouldEqual StatusCodes.OK val mapping = replacements( @@ -311,7 +286,7 @@ abstract class StorageSpec extends BaseSpec with CirceEq { } } - "query the default sparql view for files" in eventually { + "Query the default sparql view for files" in eventually { val id = s"http://delta:8080/v1/resources/$projectRef/_/attachment.json" val query = s""" @@ -370,6 +345,82 @@ abstract class StorageSpec extends BaseSpec with CirceEq { } } + "Upload files with the .custom extension" should { + val fileContent = "file content" + + def uploadCustomFile(id: String, contentType: ContentType): ((Json, HttpResponse) => Assertion) => Task[Assertion] = + deltaClient.uploadFile[Json]( + s"/files/$projectRef/$id?storage=nxv:$storageId", + fileContent, + contentType, + "file.custom", + Coyote + ) + + def assertContentType(id: String, expectedContentType: String) = + deltaClient.get[Json](s"/files/$projectRef/attachment:$id", Coyote) { (json, response) => + response.status shouldEqual StatusCodes.OK + root._mediaType.string.getOption(json) shouldEqual Some(expectedContentType) + } + + "autodetect the correct content type when no header is passed" in { + val id = "file.custom" + for { + _ <- uploadCustomFile(id, ContentTypes.NoContentType) { expectCreated } + _ <- assertContentType(id, "application/custom") + } yield succeed + } + + "assign the content-type header provided by the user" in { + val id = "file2.custom" + for { + _ <- uploadCustomFile(id, ContentTypes.`application/json`) { expectCreated } + _ <- assertContentType(id, "application/json") + } yield succeed + } + } + + "Deprecating a storage" should { + + "deprecate a storage" in { + deltaClient.delete[Json](s"/storages/$projectRef/nxv:$storageId?rev=1", Coyote) { expectOk } + } + + "reject uploading a new file against the deprecated storage" in { + deltaClient.uploadFile[Json]( + s"/files/$projectRef/${genString()}?storage=nxv:$storageId", + "", + ContentTypes.NoContentType, + "attachment3", + Coyote + ) { + expectBadRequest + } + } + + "fetch metadata" in { + val id = s"${attachmentPrefix}attachment2" + val expected = jsonContentOf( + "/kg/files/attachment2-metadata.json", + replacements( + Coyote, + "id" -> id, + "storageId" -> storageId, + "self" -> fileSelf(projectRef, id), + "storageType" -> storageType, + "projId" -> s"$projectRef", + "project" -> s"${config.deltaUri}/projects/$projectRef", + "storageType" -> storageType + ): _* + ) + + deltaClient.get[Json](s"/files/$projectRef/attachment:attachment2", Coyote) { (json, response) => + response.status shouldEqual StatusCodes.OK + filterMetadataKeys.andThen(filterKey("_location"))(json) shouldEqual expected + } + } + } + private def attachmentString(filename: String): String = { val encodedFilename = new String(Base64.getEncoder.encode(filename.getBytes(Charsets.UTF_8))) s"=?UTF-8?B?$encodedFilename?=" @@ -388,7 +439,7 @@ abstract class StorageSpec extends BaseSpec with CirceEq { contentType(response) shouldEqual expectedContentType if (compressed) { httpEncodings(response) shouldEqual Seq(HttpEncodings.gzip) - decodeGzip(content) shouldEqual contentOf("/kg/files/attachment.json") + decodeGzip(content) shouldEqual expectedContent } else content.utf8String shouldEqual expectedContent } From 0e0bbd88e1ddbb53890cc2a61ba941f153c7d670 Mon Sep 17 00:00:00 2001 From: Simon Dumas Date: Thu, 28 Sep 2023 18:13:21 +0200 Subject: [PATCH 4/9] Fix integration tests --- .../plugins/storage/files/FormDataExtractor.scala | 1 - .../epfl/bluebrain/nexus/tests/kg/EventsSpec.scala | 14 ++++++++++++-- .../nexus/tests/kg/ProjectsDeletionSpec.scala | 4 ++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala index 74e57fae96..fac9d02eef 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala @@ -74,7 +74,6 @@ object FormDataExtractor { ): IO[FileRejection, (FileDescription, BodyPartEntity)] = { val sizeLimit = Math.min(storageAvailableSpace.getOrElse(Long.MaxValue), maxFileSize) IO.deferFuture(um(entity.withSizeLimit(sizeLimit))) - .mapError(onError) .flatMap { formData => IO.fromFuture( formData.parts diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/EventsSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/EventsSpec.scala index d673d2bde0..53c235e41c 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/EventsSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/EventsSpec.scala @@ -93,6 +93,16 @@ class EventsSpec extends BaseSpec with Inspectors { val resourceId = "https://dev.nexus.test.com/simplified-resource/1" val payload = SimpleResource.sourcePayload(resourceId, 3) + val fileContent = + """|{ + | "this": ["is", "a", "test", "attachment"] + |}""".stripMargin + + val updatedFileContent = + """|{ + | "this": ["is", "a", "test", "attachment", "2"] + |}""".stripMargin + for { //ResourceCreated event _ <- deltaClient.put[Json](s"/resources/$id/_/test-resource:1", payload, BugsBunny) { (_, response) => @@ -124,7 +134,7 @@ class EventsSpec extends BaseSpec with Inspectors { //FileCreated event _ <- deltaClient.uploadFile[Json]( s"/files/$id/attachment.json", - contentOf("/kg/files/attachment.json"), + fileContent, ContentTypes.`application/json`, "attachment.json", BugsBunny @@ -134,7 +144,7 @@ class EventsSpec extends BaseSpec with Inspectors { //FileUpdated event _ <- deltaClient.uploadFile[Json]( s"/files/$id/attachment.json?rev=1", - contentOf("/kg/files/attachment2.json"), + updatedFileContent, ContentTypes.`application/json`, "attachment.json", BugsBunny diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/ProjectsDeletionSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/ProjectsDeletionSpec.scala index e8767963cb..0d94e59757 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/ProjectsDeletionSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/ProjectsDeletionSpec.scala @@ -120,14 +120,14 @@ final class ProjectsDeletionSpec extends BaseSpec with CirceEq with EitherValuab ) _ <- deltaClient.uploadFile[Json]( s"/files/$ref1/attachment.json", - contentOf("/kg/files/attachment.json"), + "some file content", ContentTypes.`application/json`, "attachment.json", Bojack )(expectCreated) _ <- deltaClient.uploadFile[Json]( s"/files/$ref2/attachment.json", - contentOf("/kg/files/attachment.json"), + "some file content", ContentTypes.`application/json`, "attachment.json", Bojack From 6d67e9fee941480010188692b6287dca3cad21f4 Mon Sep 17 00:00:00 2001 From: Simon Dumas Date: Thu, 28 Sep 2023 18:20:31 +0200 Subject: [PATCH 5/9] Add cargo back --- build.sbt | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 31dc221b39..df6e892ae4 100755 --- a/build.sbt +++ b/build.sbt @@ -216,7 +216,7 @@ lazy val kernel = project log4cats, pureconfig, scalaLogging, - munit % Test, + munit % Test, scalaTest % Test ), addCompilerPlugin(kindProjector), @@ -735,6 +735,19 @@ lazy val storage = project coverageMinimumStmtTotal := 75 ) .dependsOn(kernel) + .settings(cargo := { + import scala.sys.process._ + + val log = streams.value.log + val cmd = Process(Seq("cargo", "build", "--release"), baseDirectory.value / "permissions-fixer") + if (cmd.! == 0) { + log.success("Cargo build successful.") + (baseDirectory.value / "permissions-fixer" / "target" / "release" / "nexus-fixer") -> "bin/nexus-fixer" + } else { + log.error("Cargo build failed.") + throw new RuntimeException + } + }) .settings( name := "storage", moduleName := "storage", @@ -767,7 +780,10 @@ lazy val storage = project baseDirectory.value / "nexus-storage.jar" ), Test / testOptions += Tests.Argument(TestFrameworks.ScalaTest, "-o", "-u", "target/test-reports"), - Test / parallelExecution := false + Test / parallelExecution := false, + Universal / mappings := { + (Universal / mappings).value :+ cargo.value + } ) lazy val tests = project From d00192121c7e3a84ec59c1282b264464e96af98a Mon Sep 17 00:00:00 2001 From: Simon Dumas Date: Thu, 28 Sep 2023 18:24:13 +0200 Subject: [PATCH 6/9] Fix compilation --- .../nexus/delta/plugins/storage/files/FormDataExtractor.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala index fac9d02eef..74e57fae96 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala @@ -74,6 +74,7 @@ object FormDataExtractor { ): IO[FileRejection, (FileDescription, BodyPartEntity)] = { val sizeLimit = Math.min(storageAvailableSpace.getOrElse(Long.MaxValue), maxFileSize) IO.deferFuture(um(entity.withSizeLimit(sizeLimit))) + .mapError(onError) .flatMap { formData => IO.fromFuture( formData.parts From 360b7b7e6f9ebcecbfa0ca07b6508db41f41c5d4 Mon Sep 17 00:00:00 2001 From: Simon Dumas Date: Tue, 3 Oct 2023 14:02:55 +0200 Subject: [PATCH 7/9] Address feedback + fix test --- .../storage/files/FormDataExtractor.scala | 48 ++++++++++++------- .../resources/kg/multi-fetch/all-success.json | 4 +- .../kg/multi-fetch/limited-access.json | 4 +- .../nexus/tests/kg/MultiFetchSpec.scala | 2 +- 4 files changed, 35 insertions(+), 23 deletions(-) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala index 74e57fae96..e112d234d7 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala @@ -17,6 +17,7 @@ import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import monix.bio.IO import monix.execution.Scheduler +import scala.concurrent.Future import scala.util.Try sealed trait FormDataExtractor { @@ -73,25 +74,17 @@ object FormDataExtractor { storageAvailableSpace: Option[Long] ): IO[FileRejection, (FileDescription, BodyPartEntity)] = { val sizeLimit = Math.min(storageAvailableSpace.getOrElse(Long.MaxValue), maxFileSize) - IO.deferFuture(um(entity.withSizeLimit(sizeLimit))) - .mapError(onError) - .flatMap { formData => - IO.fromFuture( - formData.parts - .mapAsync(parallelism = 1)(describe) - .collect { case Some(values) => values } - .toMat(Sink.headOption)(Keep.right) - .run() - ).mapError { - case _: EntityStreamSizeException => - FileTooLarge(maxFileSize, storageAvailableSpace) - case th => - WrappedAkkaRejection(MalformedRequestContentRejection(th.getMessage, th)) - }.flatMap(IO.fromOption(_, InvalidMultipartFieldName(id))) - } + for { + formData <- unmarshall(entity, sizeLimit) + fileOpt <- extractFile(formData, maxFileSize, storageAvailableSpace) + file <- IO.fromOption(fileOpt, InvalidMultipartFieldName(id)) + } yield file } - private def onError(th: Throwable) = th match { + private def unmarshall(entity: HttpEntity, sizeLimit: Long) = + IO.deferFuture(um(entity.withSizeLimit(sizeLimit))).mapError(onUnmarshallingError) + + private def onUnmarshallingError(th: Throwable) = th match { case RejectionError(r) => WrappedAkkaRejection(r) case Unmarshaller.NoContentException => @@ -106,7 +99,26 @@ object FormDataExtractor { WrappedAkkaRejection(MalformedRequestContentRejection(Option(x.getMessage).getOrElse(""), x)) } - private def describe(part: FormData.BodyPart) = part match { + private def extractFile( + formData: FormData, + maxFileSize: Long, + storageAvailableSpace: Option[Long] + ): IO[FileRejection, Option[(FileDescription, BodyPartEntity)]] = IO + .fromFuture( + formData.parts + .mapAsync(parallelism = 1)(extractFile) + .collect { case Some(values) => values } + .toMat(Sink.headOption)(Keep.right) + .run() + ) + .mapError { + case _: EntityStreamSizeException => + FileTooLarge(maxFileSize, storageAvailableSpace) + case th => + WrappedAkkaRejection(MalformedRequestContentRejection(th.getMessage, th)) + } + + private def extractFile(part: FormData.BodyPart): Future[Option[(FileDescription, BodyPartEntity)]] = part match { case part if part.name == fieldName => val filename = part.filename.getOrElse("file") val contentType = detectContentType(filename, part.entity.contentType) diff --git a/tests/src/test/resources/kg/multi-fetch/all-success.json b/tests/src/test/resources/kg/multi-fetch/all-success.json index 7b173ac281..89df0d2c7c 100644 --- a/tests/src/test/resources/kg/multi-fetch/all-success.json +++ b/tests/src/test/resources/kg/multi-fetch/all-success.json @@ -22,10 +22,10 @@ "@id" : "https://bluebrain.github.io/nexus/vocabulary/file", "project" : "{{project2}}", "value" : { - "_bytes" : 47, + "_bytes" : 21, "_digest" : { "_algorithm" : "SHA-256", - "_value" : "00ff4b34e3f3695c3abcdec61cba72c2238ed172ef34ae1196bfad6a4ec23dda" + "_value" : "5ef661604649ed3d825176db889d7af0e2c1012d2ffa5080f8a220a234212517" }, "_filename" : "attachment.json", "_mediaType" : "application/json", diff --git a/tests/src/test/resources/kg/multi-fetch/limited-access.json b/tests/src/test/resources/kg/multi-fetch/limited-access.json index db3dc24fcf..9b155e68b8 100644 --- a/tests/src/test/resources/kg/multi-fetch/limited-access.json +++ b/tests/src/test/resources/kg/multi-fetch/limited-access.json @@ -13,10 +13,10 @@ "@id" : "https://bluebrain.github.io/nexus/vocabulary/file", "project" : "{{project2}}", "value" : { - "_bytes" : 47, + "_bytes" : 21, "_digest" : { "_algorithm" : "SHA-256", - "_value" : "00ff4b34e3f3695c3abcdec61cba72c2238ed172ef34ae1196bfad6a4ec23dda" + "_value" : "5ef661604649ed3d825176db889d7af0e2c1012d2ffa5080f8a220a234212517" }, "_filename" : "attachment.json", "_mediaType" : "application/json", diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/MultiFetchSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/MultiFetchSpec.scala index 7ea95a3316..c6e2e8d134 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/MultiFetchSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/MultiFetchSpec.scala @@ -34,7 +34,7 @@ class MultiFetchSpec extends BaseSpec { _ <- deltaClient.put[Json](s"/resources/$ref11/_/nxv:resource", resourcePayload, Bob)(expectCreated) _ <- deltaClient.uploadFile[Json]( s"/files/$ref12/nxv:file", - contentOf("/kg/files/attachment.json"), + """{ "content": "json" }""", ContentTypes.`application/json`, "attachment.json", Bob From 27a41bcc7230370e21ebc0960137aefd4fd1cd9d Mon Sep 17 00:00:00 2001 From: Simon Dumas Date: Tue, 3 Oct 2023 14:21:45 +0200 Subject: [PATCH 8/9] Add documentation --- .../running-nexus/configuration/index.md | 26 +++++++++++++++++++ .../docs/releases/v1.9-release-notes.md | 8 ++++++ 2 files changed, 34 insertions(+) diff --git a/docs/src/main/paradox/docs/getting-started/running-nexus/configuration/index.md b/docs/src/main/paradox/docs/getting-started/running-nexus/configuration/index.md index efe5bcea4a..255c58c2bd 100644 --- a/docs/src/main/paradox/docs/getting-started/running-nexus/configuration/index.md +++ b/docs/src/main/paradox/docs/getting-started/running-nexus/configuration/index.md @@ -127,6 +127,32 @@ Nexus Delta supports 3 types of storages: 'disk', 'amazon' (s3 compatible) and ' - For S3 compatible storages the most relevant configuration flags are the ones related to the S3 settings: `plugins.storage.storages.amazon.default-endpoint`, `plugins.storage.storages.amazon.default-access-key` and `plugins.storage.storages.amazon.default-secret-key`. - For remote disk storages the most relevant configuration flags are `plugins.storage.storages.remote-disk.default-endpoint` (the endpoint where the remote storage service is running) and `plugins.storage.storages.remote-disk.credentials` (the method to authenticate to the remote storage service). +#### File configuration + +When the media type is not provided by the user, Delta relies on automatic detection based on the file extension in order to provide one. + +From 1.9, it is possible to provide a list of extensions with an associated media type to compute the media type. + +This list can be defined at `files.media-type-detector.extensions`: +```hocon +files { + # Allows to define default media types for the given file extensions + media-type-detector { + extensions { + custom = "application/custom" + ntriples = "application/n-triples" + } + } +} +``` + +The media type resolution process follow this order stopping at the first successful step: + +* Select the `Content-Type` header from the file creation/update request +* Compare the extension to the custom list provided in the configuratio +* Fallback on akka automatic detection +* Fallback to the default value `application/octet-stream` + #### Remote storage configuration Authentication for remote storage can be specified in three different ways. The value of `plugins.storage.storages.remote-disk.credentials` can be: diff --git a/docs/src/main/paradox/docs/releases/v1.9-release-notes.md b/docs/src/main/paradox/docs/releases/v1.9-release-notes.md index 7111ad0129..500a18c6ce 100644 --- a/docs/src/main/paradox/docs/releases/v1.9-release-notes.md +++ b/docs/src/main/paradox/docs/releases/v1.9-release-notes.md @@ -148,6 +148,14 @@ These should instead be defined in the Delta configuration. @ref:[More information](../getting-started/running-nexus/configuration/index.md#remote-storage-configuration) +### Files + +The automatic detection of the media type can now be customized at the Delta level. + +NB: The media type provided by the user still prevails over automatic detection. + +@ref:[More information](../getting-started/running-nexus/configuration/index.md#file-configuration) + ### Graph analytics #### Search endpoint for Graph analytics' views From 08a0c2402047ac2b5604c44b93409c527a8b43d1 Mon Sep 17 00:00:00 2001 From: Simon Dumas Date: Tue, 3 Oct 2023 16:37:28 +0200 Subject: [PATCH 9/9] Remove duplicate test --- .../storage/attributes/ContentTypeDetectorSuite.scala | 7 ------- 1 file changed, 7 deletions(-) diff --git a/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/attributes/ContentTypeDetectorSuite.scala b/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/attributes/ContentTypeDetectorSuite.scala index 3bbd428070..c9a9815413 100644 --- a/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/attributes/ContentTypeDetectorSuite.scala +++ b/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/attributes/ContentTypeDetectorSuite.scala @@ -25,13 +25,6 @@ class ContentTypeDetectorSuite extends FunSuite { assertEquals(detector(jsonPath, isDir = false), expected) } - test("Detect overridden content type") { - val customMediaType = MediaTypes.`application/vnd.api+json` - val detector = new ContentTypeDetector(MediaTypeDetectorConfig("json" -> MediaTypes.`application/vnd.api+json`)) - val expected = ContentType(customMediaType, () => `UTF-8`) - assertEquals(detector(jsonPath, isDir = false), expected) - } - test("Detect `application/octet-stream` as a default value") { val detector = new ContentTypeDetector(MediaTypeDetectorConfig("json" -> MediaTypes.`application/vnd.api+json`)) val expected = ContentTypes.`application/octet-stream`