From eabbea59b0b0dcf9dfc80fbaba76554a5e09157b Mon Sep 17 00:00:00 2001 From: Oliver <20188437+olivergrabinski@users.noreply.github.com> Date: Tue, 27 Feb 2024 16:51:02 +0100 Subject: [PATCH] Allow custom metadata when linking a file (#4758) --- .../storage/files/FormDataExtractor.scala | 13 +-- .../storage/files/model/FileAttributes.scala | 7 +- .../files/model/FileCustomMetadata.scala | 23 +++++ .../storage/files/model/FileDescription.scala | 31 ++++--- .../storage/files/model/FileRejection.scala | 14 ++- .../storage/files/routes/FilesRoutes.scala | 88 +++++++++---------- .../remote/RemoteDiskStorageCopyFiles.scala | 9 +- .../plugins/storage/files/FilesSpec.scala | 60 +++++++++++-- .../storage/files/FormDataExtractorSpec.scala | 4 +- .../files/routes/FilesRoutesSpec.scala | 8 +- .../RemoteStorageClientFixtures.scala | 4 +- .../sdk/directives/DeltaDirectives.scala | 10 --- .../delta/api/assets/files/create-post.sh | 1 + .../docs/delta/api/assets/files/create-put.sh | 1 + .../delta/api/assets/files/created-post.json | 1 + .../delta/api/assets/files/created-put.json | 1 + .../delta/api/assets/files/link-post.json | 10 ++- .../docs/delta/api/assets/files/link-post.sh | 10 ++- .../docs/delta/api/assets/files/link-put.json | 10 ++- .../docs/delta/api/assets/files/link-put.sh | 10 ++- .../delta/api/assets/files/linked-post.json | 6 ++ .../delta/api/assets/files/linked-put.json | 6 ++ .../main/paradox/docs/delta/api/files-api.md | 17 +++- .../tests/kg/files/RemoteStorageSpec.scala | 87 ++++++++++++++++++ 24 files changed, 326 insertions(+), 105 deletions(-) create mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileCustomMetadata.scala diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala index a08930867e..d6b6e84250 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala @@ -13,12 +13,11 @@ import cats.syntax.all._ import ch.epfl.bluebrain.nexus.delta.kernel.error.NotARejection import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig import ch.epfl.bluebrain.nexus.delta.kernel.utils.FileUtils -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection.{FileTooLarge, InvalidCustomMetadata, InvalidMultipartFieldName, WrappedAkkaRejection} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileCustomMetadata, FileRejection} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label -import io.circe.generic.semiauto.deriveDecoder -import io.circe.{parser, Decoder} +import io.circe.parser import scala.concurrent.{ExecutionContext, Future} import scala.util.Try @@ -157,14 +156,6 @@ object FormDataExtractor { part.entity.discardBytes().future.as(None) } - private case class FileCustomMetadata( - name: Option[String], - description: Option[String], - keywords: Option[Map[Label, String]] - ) - implicit private val fileUploadMetadataDecoder: Decoder[FileCustomMetadata] = - deriveDecoder[FileCustomMetadata] - private def extractMetadata( part: Multipart.FormData.BodyPart ): Either[FileRejection, FileCustomMetadata] = { diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileAttributes.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileAttributes.scala index 9aeba03e56..fbe8f57f59 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileAttributes.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileAttributes.scala @@ -63,15 +63,16 @@ trait LimitedFileAttributes { object FileAttributes { def from(description: FileDescription, storageMetadata: FileStorageMetadata): FileAttributes = { + val customMetadata = description.metadata.getOrElse(FileCustomMetadata.empty) FileAttributes( storageMetadata.uuid, storageMetadata.location, storageMetadata.path, description.filename, description.mediaType, - description.keywords, - description.description, - description.name, + customMetadata.keywords.getOrElse(Map.empty), + customMetadata.description, + customMetadata.name, storageMetadata.bytes, storageMetadata.digest, storageMetadata.origin diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileCustomMetadata.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileCustomMetadata.scala new file mode 100644 index 0000000000..a43cef64be --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileCustomMetadata.scala @@ -0,0 +1,23 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model + +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label +import io.circe.Decoder +import io.circe.generic.semiauto.deriveDecoder + +/** + * Custom metadata for a file that can be specified by the user. + */ +case class FileCustomMetadata( + name: Option[String], + description: Option[String], + keywords: Option[Map[Label, String]] +) + +object FileCustomMetadata { + + implicit val fileUploadMetadataDecoder: Decoder[FileCustomMetadata] = + deriveDecoder[FileCustomMetadata] + + val empty: FileCustomMetadata = FileCustomMetadata(None, None, None) + +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileDescription.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileDescription.scala index 766015d807..0f7ac3eae5 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileDescription.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileDescription.scala @@ -1,15 +1,13 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model import akka.http.scaladsl.model.ContentType +import cats.implicits.catsSyntaxOptionId import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.UploadedFileInformation -import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label case class FileDescription( filename: String, - keywords: Map[Label, String], mediaType: Option[ContentType], - description: Option[String], - name: Option[String] + metadata: Option[FileCustomMetadata] ) object FileDescription { @@ -17,17 +15,26 @@ object FileDescription { from(file.attributes) } - def from(fileAttributes: FileAttributes): FileDescription = { + def from(fileAttributes: FileAttributes): FileDescription = FileDescription( fileAttributes.filename, - fileAttributes.keywords, fileAttributes.mediaType, - fileAttributes.description, - fileAttributes.name + FileCustomMetadata( + fileAttributes.name, + fileAttributes.description, + Some(fileAttributes.keywords) + ).some + ) + + def from(info: UploadedFileInformation): FileDescription = + FileDescription( + info.filename, + Some(info.suppliedContentType), + FileCustomMetadata( + info.name, + info.description, + Some(info.keywords) + ).some ) - } - def from(info: UploadedFileInformation): FileDescription = { - FileDescription(info.filename, info.keywords, Some(info.suppliedContentType), info.description, info.name) - } } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileRejection.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileRejection.scala index 50d51cb781..53938e818d 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileRejection.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileRejection.scala @@ -227,7 +227,19 @@ object FileRejection { * the rejection which occurred with the storage */ final case class LinkRejection(id: Iri, storageId: Iri, rejection: StorageFileRejection) - extends FileRejection(s"File '$id' could not be linked using storage '$storageId'", Some(rejection.loggedDetails)) + extends FileRejection( + s"File '$id' could not be linked using storage '$storageId'", + Some(rejection.loggedDetails) + ) + + /** + * Rejection returned when attempting to link a file without providing a filename or a path that ends with a + * filename. + */ + final case object InvalidFileLink + extends FileRejection( + s"Linking a file cannot be performed without a 'filename' or a 'path' that does not end with a filename." + ) final case class CopyRejection( sourceProj: ProjectRef, diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala index b10bbc8486..61e6c4cc84 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala @@ -1,6 +1,5 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes -import akka.http.scaladsl.model.MediaTypes.`multipart/form-data` import akka.http.scaladsl.model.StatusCodes.Created import akka.http.scaladsl.model.Uri.Path import akka.http.scaladsl.model.headers.Accept @@ -9,8 +8,9 @@ import akka.http.scaladsl.server._ import cats.effect.IO import cats.syntax.all._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection._ -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{File, FileDescription, FileId, FileRejection} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.permissions.{read => Read, write => Write} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.FilesRoutes.LinkFileRequest.{fileDescriptionFromRequest, linkFileDecoder} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.FilesRoutes._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{schemas, FileResource, Files} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.ShowFileLocation @@ -28,7 +28,6 @@ import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.implicits._ import ch.epfl.bluebrain.nexus.delta.sdk.model.routes.Tag import ch.epfl.bluebrain.nexus.delta.sdk.model.{BaseUri, IdSegment} -import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag import io.circe.Decoder import io.circe.generic.extras.Configuration @@ -85,17 +84,20 @@ final class FilesRoutes( operationName(s"$prefixSegment/files/{org}/{project}") { concat( // Link a file without id segment - entity(as[LinkFile]) { case LinkFile(path, description) => + entity(as[LinkFileRequest]) { linkRequest => emit( Created, - files - .createLink(storage, project, description, path, tag) - .index(mode) + fileDescriptionFromRequest(linkRequest) + .flatMap { desc => + files + .createLink(storage, project, desc, linkRequest.path, tag) + .index(mode) + } .attemptNarrow[FileRejection] ) }, // Create a file without id segment - (contentType(`multipart/form-data`) & extractRequestEntity) { entity => + extractRequestEntity { entity => emit( Created, files.create(storage, project, entity, tag).index(mode).attemptNarrow[FileRejection] @@ -116,11 +118,21 @@ final class FilesRoutes( case (rev, storage, tag) => concat( // Update a Link - entity(as[LinkFile]) { case LinkFile(path, description) => + entity(as[LinkFileRequest]) { linkRequest => emit( - files - .updateLink(fileId, storage, description, path, rev, tag) - .index(mode) + fileDescriptionFromRequest(linkRequest) + .flatMap { description => + files + .updateLink( + fileId, + storage, + description, + linkRequest.path, + rev, + tag + ) + .index(mode) + } .attemptNarrow[FileRejection] ) }, @@ -138,12 +150,15 @@ final class FilesRoutes( parameters("storage".as[IdSegment].?, "tag".as[UserTag].?) { case (storage, tag) => concat( // Link a file with id segment - entity(as[LinkFile]) { case LinkFile(path, description) => + entity(as[LinkFileRequest]) { linkRequest => emit( Created, - files - .createLink(fileId, storage, description, path, tag) - .index(mode) + fileDescriptionFromRequest(linkRequest) + .flatMap { description => + files + .createLink(fileId, storage, description, linkRequest.path, tag) + .index(mode) + } .attemptNarrow[FileRejection] ) }, @@ -279,37 +294,18 @@ object FilesRoutes { path: Path, filename: Option[String], mediaType: Option[ContentType], - keywords: Map[Label, String] = Map.empty, - description: Option[String], - name: Option[String] + metadata: Option[FileCustomMetadata] ) - final case class LinkFile(path: Path, fileDescription: FileDescription) - object LinkFile { + + object LinkFileRequest { @nowarn("cat=unused") - implicit private val config: Configuration = Configuration.default.withStrictDecoding.withDefaults - implicit val linkFileDecoder: Decoder[LinkFile] = { - deriveConfiguredDecoder[LinkFileRequest] - .flatMap { case LinkFileRequest(path, filename, mediaType, keywords, description, name) => - filename.orElse(path.lastSegment) match { - case Some(derivedFilename) => - Decoder.const( - LinkFile( - path, - FileDescription( - derivedFilename, - keywords, - mediaType, - description, - name - ) - ) - ) - case None => - Decoder.failedWithMessage( - "Linking a file cannot be performed without a 'filename' or a 'path' that does not end with a filename." - ) - } - } - } + implicit private val config: Configuration = Configuration.default + implicit val linkFileDecoder: Decoder[LinkFileRequest] = deriveConfiguredDecoder[LinkFileRequest] + + def fileDescriptionFromRequest(f: LinkFileRequest): IO[FileDescription] = + f.filename.orElse(f.path.lastSegment) match { + case Some(value) => IO.pure(FileDescription(value, f.mediaType, f.metadata)) + case None => IO.raiseError(InvalidFileLink) + } } } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFiles.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFiles.scala index 4fc3e52ade..c45f9d835c 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFiles.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFiles.scala @@ -4,7 +4,7 @@ import akka.http.scaladsl.model.Uri import akka.http.scaladsl.model.Uri.Path import cats.data.NonEmptyList import cats.effect.IO -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileCustomMetadata} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.RemoteDiskStorage import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.SaveFile.intermediateFolders import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient @@ -42,15 +42,16 @@ object RemoteDiskStorageCopyFiles { ): FileAttributes = { val sourceFileMetadata = cd.sourceMetadata val sourceFileDescription = cd.sourceUserSuppliedMetadata + val customMetadata = sourceFileDescription.metadata.getOrElse(FileCustomMetadata.empty) FileAttributes( uuid = cd.destUuid, location = absoluteDestPath, path = relativeDestPath, filename = sourceFileDescription.filename, mediaType = sourceFileDescription.mediaType, - keywords = sourceFileDescription.keywords, - description = sourceFileDescription.description, - name = sourceFileDescription.name, + keywords = customMetadata.keywords.getOrElse(Map.empty), + description = customMetadata.description, + name = customMetadata.name, bytes = sourceFileMetadata.bytes, digest = sourceFileMetadata.digest, origin = sourceFileMetadata.origin diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala index 696f63250a..40f2f18bc0 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala @@ -3,17 +3,16 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files import akka.actor.typed.scaladsl.adapter._ import akka.actor.{typed, ActorSystem} import akka.http.scaladsl.model.ContentTypes.`text/plain(UTF-8)` -import akka.http.scaladsl.model.Uri +import akka.http.scaladsl.model.{ContentType, Uri} import akka.testkit.TestKit import cats.effect.IO -import akka.http.scaladsl.model.ContentType import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig import ch.epfl.bluebrain.nexus.delta.plugins.storage.RemoteContextResolutionFixture import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.NotComputedDigest import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Storage import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection._ -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileDescription, FileId} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileCustomMetadata, FileDescription, FileId} import ch.epfl.bluebrain.nexus.delta.plugins.storage.remotestorage.RemoteStorageClientFixtures import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection.StorageNotFound import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageType.{RemoteDiskStorage => RemoteStorageType} @@ -62,13 +61,24 @@ class FilesSpec(fixture: RemoteStorageClientFixtures) private val alice = User("Alice", realm) def description(filename: String): FileDescription = { - FileDescription(filename, Map.empty, None, None, None) + FileDescription(filename, None, Some(FileCustomMetadata.empty)) } def description(filename: String, contentType: ContentType): FileDescription = { - FileDescription(filename, Map.empty, Some(contentType), None, None) + FileDescription(filename, Some(contentType), Some(FileCustomMetadata.empty)) } + def descriptionWithName(filename: String, name: String): FileDescription = + FileDescription(filename, None, Some(FileCustomMetadata(Some(name), None, None))) + + def descriptionWithMetadata( + filename: String, + name: String, + description: String, + keywords: Map[Label, String] + ): FileDescription = + FileDescription(filename, None, Some(FileCustomMetadata(Some(name), Some(description), Some(keywords)))) + "The Files operations bundle" when { implicit val typedSystem: typed.ActorSystem[Nothing] = system.toTyped implicit val caller: Caller = Caller(bob, Set(bob, Group("mygroup", realm), Authenticated(realm))) @@ -289,6 +299,21 @@ class FilesSpec(fixture: RemoteStorageClientFixtures) fileByTag.value.tags.tags should contain(tag) } + "succeed with custom user provided metadata" in { + val (name, description, keywords) = (genString(), genString(), genKeywords()) + val fileDescription = descriptionWithMetadata("file-5.txt", name, description, keywords) + + val id = fileId(genString()) + val path = Uri.Path(s"my/file-5.txt") + + files.createLink(id, Some(remoteId), fileDescription, path, None).accepted + val fetchedFile = files.fetch(id).accepted + + fetchedFile.value.attributes.name should contain(name) + fetchedFile.value.attributes.description should contain(description) + fetchedFile.value.attributes.keywords shouldEqual keywords + } + "reject if file id already exists" in { files .createLink(fileId("file2"), Some(remoteId), description("myfile.txt"), Uri.Path.Empty, None) @@ -405,6 +430,31 @@ class FilesSpec(fixture: RemoteStorageClientFixtures) byTag shouldEqual expected } + "succeed if also updating custom metadata" in { + val id = fileId(genString()) + val path = Uri.Path("my/file-6.txt") + + val (name, desc, keywords) = (genString(), genString(), genKeywords()) + + val originalFileDescription = description("file-6.txt") + val updatedFileDescription = descriptionWithMetadata("file-6.txt", name, desc, keywords) + + files.createLink(id, Some(remoteId), originalFileDescription, path, None).accepted + + val fetched = files.fetch(id).accepted + files.updateAttributes(fetched.id, projectRef).accepted + files.updateLink(id, Some(remoteId), updatedFileDescription, path, 2, None) + + eventually { + files.fetch(id).map { fetched => + fetched.value.attributes.name should contain(name) + fetched.value.attributes.description should contain(desc) + fetched.value.attributes.keywords shouldEqual keywords + } + } + + } + "reject if file doesn't exists" in { files .updateLink(fileIdIri(nxv + "other"), None, description("myfile.txt"), Uri.Path.Empty, 1, None) diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractorSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractorSpec.scala index 93a7858f5a..93739fb61b 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractorSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractorSpec.scala @@ -140,8 +140,7 @@ class FormDataExtractorSpec "fail to be extracted if the custom user metadata has invalid keywords" in { val entity = entityWithKeywords(KeyThatIsTooLong := "value") - val rej = extractor(iri, entity, 2000, None).rejectedWith[InvalidCustomMetadata] - println(rej) + extractor(iri, entity, 2000, None).rejectedWith[InvalidCustomMetadata] } "fail to be extracted if no file part exists found" in { @@ -151,7 +150,6 @@ class FormDataExtractorSpec "fail to be extracted if payload size is too large" in { val entity = createEntity("other", `text/plain(UTF-8)`, None) - extractor(iri, entity, 10, None).rejected shouldEqual FileTooLarge(10L, None) } } diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala index 89057a1d02..bdc1543dc4 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala @@ -220,7 +220,13 @@ class FilesRoutesSpec postJson("/v1/files/org/proj", payload) ~> asWriter ~> routes ~> check { status shouldEqual StatusCodes.BadRequest response.asJson shouldEqual - jsonContentOf("files/errors/file-link-no-filename.json") + json""" + { + "@context" : "https://bluebrain.github.io/nexus/contexts/error.json", + "@type" : "InvalidFileLink", + "reason" : "Linking a file cannot be performed without a 'filename' or a 'path' that does not end with a filename." + } + """ } } diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/remotestorage/RemoteStorageClientFixtures.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/remotestorage/RemoteStorageClientFixtures.scala index 38aa0ad3e7..0becb45de6 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/remotestorage/RemoteStorageClientFixtures.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/remotestorage/RemoteStorageClientFixtures.scala @@ -21,7 +21,7 @@ trait RemoteStorageClientFixtures extends BeforeAndAfterAll with ConfigFixtures private val rwx = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwxrwxrwx")) private val tmpFolder: Path = Files.createTempDirectory("root", rwx) - val storageVersion: String = "1.8.0-M12" + val storageVersion: String = "1.9.0" protected val container: RemoteStorageContainer = new RemoteStorageContainer(storageVersion, tmpFolder) @@ -46,7 +46,7 @@ trait RemoteStorageClientFixtures extends BeforeAndAfterAll with ConfigFixtures val bucketNexus = Files.createDirectory(bucket.resolve("nexus"), rwx) val my = Files.createDirectory(bucket.resolve("my"), rwx) - (1 to 4).map(idx => s"file-$idx.txt").foreach { fileName => + (1 to 6).map(idx => s"file-$idx.txt").foreach { fileName => val path = Files.createFile(my.resolve(fileName), rwx) path.toFile.setWritable(true, false) Files.writeString(path, "file content") diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/directives/DeltaDirectives.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/directives/DeltaDirectives.scala index f7c1085fe0..11bf717f3b 100644 --- a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/directives/DeltaDirectives.scala +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/directives/DeltaDirectives.scala @@ -114,16 +114,6 @@ trait DeltaDirectives extends UriDirectives { } } - def contentType(mediaType: MediaType): Directive0 = { - headerValueByName("Content-Type").flatMap { contentType => - if (contentType == mediaType.value) { - pass - } else { - reject() - } - } - } - /** * If the `Accept` header is set to `text/html`, redirect to the matching resource page in fusion if the feature is * enabled diff --git a/docs/src/main/paradox/docs/delta/api/assets/files/create-post.sh b/docs/src/main/paradox/docs/delta/api/assets/files/create-post.sh index 6231d071b8..2f30691272 100644 --- a/docs/src/main/paradox/docs/delta/api/assets/files/create-post.sh +++ b/docs/src/main/paradox/docs/delta/api/assets/files/create-post.sh @@ -1,3 +1,4 @@ curl -X POST \ -F "file=@/path/to/myfile.jpg;type=image/jpeg" \ + -F 'metadata="{\"name\": \"My File\"}"' \ "http://localhost:8080/v1/files/myorg/myproject" \ No newline at end of file diff --git a/docs/src/main/paradox/docs/delta/api/assets/files/create-put.sh b/docs/src/main/paradox/docs/delta/api/assets/files/create-put.sh index e608dcaeb0..c175959547 100644 --- a/docs/src/main/paradox/docs/delta/api/assets/files/create-put.sh +++ b/docs/src/main/paradox/docs/delta/api/assets/files/create-put.sh @@ -1,3 +1,4 @@ curl -X PUT \ -F "file=@/path/to/myfile.pdf;type=application/pdf" \ + -F 'metadata="{\"name\": \"My File\"}"' \ "http://localhost:8080/v1/files/myorg/myproject/myfile?storage=remote&tag=mytag" \ No newline at end of file diff --git a/docs/src/main/paradox/docs/delta/api/assets/files/created-post.json b/docs/src/main/paradox/docs/delta/api/assets/files/created-post.json index 0a9246258f..ea55f61708 100644 --- a/docs/src/main/paradox/docs/delta/api/assets/files/created-post.json +++ b/docs/src/main/paradox/docs/delta/api/assets/files/created-post.json @@ -5,6 +5,7 @@ ], "@id": "http://localhost:8080/v1/resources/myorg/myproject/_/c581fdd4-6151-4430-aa33-f97ab6aa0b38", "@type": "File", + "name": "My File", "_bytes": 8615, "_constrainedBy": "https://bluebrain.github.io/nexus/schemas/files.json", "_createdAt": "2021-05-12T07:28:30.472Z", diff --git a/docs/src/main/paradox/docs/delta/api/assets/files/created-put.json b/docs/src/main/paradox/docs/delta/api/assets/files/created-put.json index 515cc415f4..888127df3b 100644 --- a/docs/src/main/paradox/docs/delta/api/assets/files/created-put.json +++ b/docs/src/main/paradox/docs/delta/api/assets/files/created-put.json @@ -5,6 +5,7 @@ ], "@id": "http://localhost:8080/v1/resources/myorg/myproject/_/myfile", "@type": "File", + "name": "My File", "_bytes": 5963969, "_constrainedBy": "https://bluebrain.github.io/nexus/schemas/files.json", "_createdAt": "2021-05-12T07:30:54.576Z", diff --git a/docs/src/main/paradox/docs/delta/api/assets/files/link-post.json b/docs/src/main/paradox/docs/delta/api/assets/files/link-post.json index 446409ecd5..194a0dbbdc 100644 --- a/docs/src/main/paradox/docs/delta/api/assets/files/link-post.json +++ b/docs/src/main/paradox/docs/delta/api/assets/files/link-post.json @@ -1,5 +1,13 @@ { "path": "relative/path/to/myfile.png", "filename": "myfile.png", - "mediaType": "image/png" + "mediaType": "image/png", + "metadata": { + "name": "My File", + "description": "a description of the file", + "keywords": { + "key1": "value1", + "key2": "value2" + } + } } \ No newline at end of file diff --git a/docs/src/main/paradox/docs/delta/api/assets/files/link-post.sh b/docs/src/main/paradox/docs/delta/api/assets/files/link-post.sh index 1e4e51d762..4d3d7dc4f3 100644 --- a/docs/src/main/paradox/docs/delta/api/assets/files/link-post.sh +++ b/docs/src/main/paradox/docs/delta/api/assets/files/link-post.sh @@ -4,5 +4,13 @@ curl -X POST \ '{ "path": "relative/path/to/myfile.png", "filename": "myfile.png", - "mediaType": "image/png" + "mediaType": "image/png", + "metadata": { + "name": "My File", + "description": "a description of the file", + "keywords": { + "key1": "value1", + "key2": "value2" + } + } }' \ No newline at end of file diff --git a/docs/src/main/paradox/docs/delta/api/assets/files/link-put.json b/docs/src/main/paradox/docs/delta/api/assets/files/link-put.json index 1ab60517e2..73e0e5235f 100644 --- a/docs/src/main/paradox/docs/delta/api/assets/files/link-put.json +++ b/docs/src/main/paradox/docs/delta/api/assets/files/link-put.json @@ -1,3 +1,11 @@ { - "path": "relative/path/to/myfile2.png" + "path": "relative/path/to/myfile2.png", + "metadata": { + "name": "My File", + "description": "a description of the file", + "keywords": { + "key1": "value1", + "key2": "value2" + } + } } \ No newline at end of file diff --git a/docs/src/main/paradox/docs/delta/api/assets/files/link-put.sh b/docs/src/main/paradox/docs/delta/api/assets/files/link-put.sh index ca815cc4e8..4ba756ce25 100644 --- a/docs/src/main/paradox/docs/delta/api/assets/files/link-put.sh +++ b/docs/src/main/paradox/docs/delta/api/assets/files/link-put.sh @@ -2,5 +2,13 @@ curl -X PUT \ -H "Content-Type: application/json" \ "http://localhost:8080/v1/files/myorg/myproject/mylink?storage=remote&tag=mytag" -d \ '{ - "path": "relative/path/to/myfile2.png" + "path": "relative/path/to/myfile2.png", + "metadata": { + "name": "My File", + "description": "a description of the file", + "keywords": { + "key1": "value1", + "key2": "value2" + } + } }' \ No newline at end of file diff --git a/docs/src/main/paradox/docs/delta/api/assets/files/linked-post.json b/docs/src/main/paradox/docs/delta/api/assets/files/linked-post.json index 73a2252ae8..2f74efcad9 100644 --- a/docs/src/main/paradox/docs/delta/api/assets/files/linked-post.json +++ b/docs/src/main/paradox/docs/delta/api/assets/files/linked-post.json @@ -5,6 +5,8 @@ ], "@id": "http://localhost:8080/v1/resources/myorg/myproject/_/9ca0d270-35f0-4369-9c17-296fe36fd9a5", "@type": "File", + "name": "My File", + "description": "a description of the file", "_bytes": 1658857, "_constrainedBy": "https://bluebrain.github.io/nexus/schemas/files.json", "_createdAt": "2021-05-12T07:56:42.991Z", @@ -15,6 +17,10 @@ }, "_filename": "myfile.png", "_incoming": "http://localhost:8080/v1/files/myorg/myproject/9ca0d270-35f0-4369-9c17-296fe36fd9a5/incoming", + "_keywords": { + "key1": "value1", + "key2": "value2" + }, "_location": "file:///tmp/test/nexus/myorg/myproject/3/4/c/4/9/1/a/1/myfile.png", "_mediaType": "image/png", "_origin": "Storage", diff --git a/docs/src/main/paradox/docs/delta/api/assets/files/linked-put.json b/docs/src/main/paradox/docs/delta/api/assets/files/linked-put.json index 741aa635c5..6c4364e3b7 100644 --- a/docs/src/main/paradox/docs/delta/api/assets/files/linked-put.json +++ b/docs/src/main/paradox/docs/delta/api/assets/files/linked-put.json @@ -5,6 +5,8 @@ ], "@id": "http://localhost:8080/v1/resources/myorg/myproject/_/mylink", "@type": "File", + "name": "My File", + "description": "description of the file", "_bytes": 1658857, "_constrainedBy": "https://bluebrain.github.io/nexus/schemas/files.json", "_createdAt": "2021-05-12T08:00:34.563Z", @@ -15,6 +17,10 @@ }, "_filename": "myfile2.png", "_incoming": "http://localhost:8080/v1/files/myorg/myproject/mylink/incoming", + "_keywords": { + "key1": "value1", + "key2": "value2" + }, "_location": "file:///tmp/test/nexus/myorg/myproject/6/6/0/4/9/9/d/9/myfile2.png", "_origin": "Storage", "_outgoing": "http://localhost:8080/v1/files/myorg/myproject/mylink/outgoing", diff --git a/docs/src/main/paradox/docs/delta/api/files-api.md b/docs/src/main/paradox/docs/delta/api/files-api.md index f3d66cfb5a..da24dba1f6 100644 --- a/docs/src/main/paradox/docs/delta/api/files-api.md +++ b/docs/src/main/paradox/docs/delta/api/files-api.md @@ -22,6 +22,7 @@ When using the endpoints described on this page, the responses will contain glob - `_bytes`: size of the file in bytes - `_digest`: algorithm and checksum used for file integrity - `_filename`: name of the file +- `_keywords`: list of keywords associated with the file and which can be used to search for the file - `_location`: path where the file is stored on the underlying storage - `_mediaType`: @link:[MIME](https://en.wikipedia.org/wiki/MIME){ open=new } specifying the type of the file - `_origin`: whether the file attributes resulted from an action taken by a client or the Nexus Storage Service @@ -88,7 +89,7 @@ The body should be a multipart form, to allow file upload. The form should conta This part can contain the following disposition parameters: - `filename`: the filename which will be used in the back-end file system -- `metadata`: a JSON object containing the following one or more of the following fields: +- `metadata`: a JSON object containing one or more of the following fields: - `name`: a string which is a descriptive name for the file. It will be indexed in the full-text search. - `description`: a string that describes the file. It will be indexed in the full-text search. - `keywords`: a JSON object with `Label` keys and `string` values. These keywords will be indexed and can be used to search for the file. @@ -110,7 +111,8 @@ POST /v1/files/{org_label}/{project_label}?storage={storageId}&tag={tagName} { "path": "{path}", "filename": "{filename}", - "mediaType": "{mediaType}" + "mediaType": "{mediaType}", + "metadata": "{metadata}" } ``` @@ -120,6 +122,10 @@ POST /v1/files/{org_label}/{project_label}?storage={storageId}&tag={tagName} - `{filename}`: String - the name that will be given to the file during linking. This field is optional. When not specified, the original filename is retained. - `{mediaType}`: String - the MediaType fo the file. This field is optional. When not specified, Nexus Delta will attempt to detect it. - `{tagName}` an optional label given to the linked file resource on its first revision. +- `{metadata}`: JSON Object - optional object containing one or more of the following fields: + - `name`: a string which is a descriptive name for the file. It will be indexed in the full-text search. + - `description`: a string that describes the file. It will be indexed in the full-text search. + - `keywords`: a JSON object with `Label` keys and `string` values. These keywords will be indexed and can be used to search for the file. **Example** @@ -143,7 +149,8 @@ PUT /v1/files/{org_label}/{project_label}/{file_id}?storage={storageId}&tag={tag { "path": "{path}", "filename": "{filename}", - "mediaType": "{mediaType}" + "mediaType": "{mediaType}", + "metadata": "{metadata}" } ``` @@ -155,6 +162,10 @@ When not specified, the default storage of the project is used. - `{filename}`: String - the name that will be given to the file during linking. This field is optional. When not specified, the original filename is retained. - `{mediaType}`: String - the MediaType fo the file. This field is optional. When not specified, Nexus Delta will attempt to detect it. - `{tagName}` an optional label given to the linked file resource on its first revision. +- `{metadata}`: JSON Object - optional object containing one or more of the following fields: + - `name`: a string which is a descriptive name for the file. It will be indexed in the full-text search. + - `description`: a string that describes the file. It will be indexed in the full-text search. + - `keywords`: a JSON object with `Label` keys and `string` values. These keywords will be indexed and can be used to search for the file. **Example** diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/RemoteStorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/RemoteStorageSpec.scala index dd4ba58a12..872e7ca484 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/RemoteStorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/RemoteStorageSpec.scala @@ -105,6 +105,10 @@ class RemoteStorageSpec extends StorageSpec { sb.toString } + case class CustomMetadata(name: String, description: String, keywords: Map[String, String]) + private val customMetadata = + CustomMetadata("cool name", "good description", Map("key1" -> "value1", "key2" -> "value2")) + "succeed many large files are in the archive, going over the time limit" ignore { val content = randomString(130000000) val payload = jsonContentOf("kg/archives/archive-many-large-files.json") @@ -162,6 +166,22 @@ class RemoteStorageSpec extends StorageSpec { "mediaType" := mediaType ) + private def linkPayloadWithMetadata( + filename: String, + path: String, + md: CustomMetadata + ) = + Json.obj( + "filename" := filename, + "path" := path, + "mediaType" := None, + "metadata" := Json.obj( + "name" := md.name, + "description" := md.description, + "keywords" := md.keywords + ) + ) + def linkFile(payload: Json)(fileId: String, filename: String, mediaType: Option[String]) = { val expected = jsonContentOf( "kg/files/remote-linked.json", @@ -311,6 +331,43 @@ class RemoteStorageSpec extends StorageSpec { } + "Linking a file with custom metadata should" should { + + "succeed" in { + val filename = s"${genString()}.txt" + val md = customMetadata + val payload = linkPayloadWithMetadata(filename, filename, md) + + for { + _ <- createFileInStorageService(filename) + _ <- linkFile(filename, payload) + _ <- assertCorrectCustomMetadata(filename, md) + } yield succeed + } + + "succeed when updating with metadata" in { + val filename = s"${genString()}.txt" + val filename2 = s"${genString()}.txt" + val md = customMetadata + + val simplePayload = linkPayload(filename, filename, None) + val payloadWithMetadata = linkPayloadWithMetadata(filename2, filename2, md) + + val setup = for { + _ <- createFileInStorageService(filename) + _ <- createFileInStorageService(filename2) + _ <- linkFile(filename, simplePayload) + } yield succeed + + setup.accepted + eventually { assertFileRevision(filename, 2) } + updateFileLink(filename, payloadWithMetadata).accepted + + eventually { assertCorrectCustomMetadata(filename, md) } + } + + } + "The file-attributes-updated projection description" should { "exist" in { aclDsl.addPermission("/", Coyote, Supervision.Read).accepted @@ -364,4 +421,34 @@ class RemoteStorageSpec extends StorageSpec { } } + + private def linkFile(filename: String, payload: Json) = + deltaClient.put[Json](s"/files/$projectRef/$filename?storage=nxv:${storageId}2", payload, Coyote) { (_, response) => + response.status shouldEqual StatusCodes.Created + } + + private def assertFileRevision(filename: String, expectedRev: Int) = + deltaClient + .get[Json](s"/files/$projectRef/$filename", Coyote) { (json, _) => + json.hcursor.get[Int]("_rev").toOption should contain(expectedRev) + } + + private def updateFileLink(filename: String, payload: Json) = + deltaClient + .put[Json](s"/files/$projectRef/$filename?rev=2&storage=nxv:${storageId}2", payload, Coyote) { (_, response) => + response.status shouldEqual StatusCodes.OK + } + + private def assertCorrectCustomMetadata( + filename: String, + customMetadata: CustomMetadata + ) = + deltaClient + .get[Json](s"/files/$projectRef/$filename", Coyote) { (json, response) => + response.status shouldEqual StatusCodes.OK + json.hcursor.get[String]("name").toOption should contain(customMetadata.name) + json.hcursor.get[String]("description").toOption should contain(customMetadata.description) + json.hcursor.get[Map[String, String]]("_keywords").toOption should contain(customMetadata.keywords) + } + }