diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala index 20c89c996a..4c346a954f 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala @@ -449,7 +449,7 @@ final class Files( filename: String, fileId: Iri ): IO[FileStorageMetadata] = - LinkFile(storage, remoteDiskStorageClient, config) + LinkFile(storage, remoteDiskStorageClient) .apply(path, filename) .adaptError { case e: StorageFileRejection => LinkRejection(fileId, storage.id, e) } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala index 323dfef755..f0c4992914 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala @@ -9,7 +9,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{DiskStorageFetchFile, DiskStorageSaveFile} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.{S3StorageFetchFile, S3StorageLinkFile, S3StorageSaveFile} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.{S3StorageFetchFile, S3StorageSaveFile} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{contexts, Storages} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.ContextValue @@ -102,10 +102,6 @@ object Storage { def saveFile(config: StorageTypeConfig)(implicit as: ActorSystem, uuidf: UUIDF): SaveFile = new S3StorageSaveFile(this, config) - - def linkFile(config: StorageTypeConfig)(implicit as: ActorSystem, uuidf: UUIDF): LinkFile = - new S3StorageLinkFile(this, config) - } /** diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/LinkFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/LinkFile.scala index 447e2ade86..b540a3d70b 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/LinkFile.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/LinkFile.scala @@ -1,11 +1,9 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations -import akka.actor.ActorSystem import akka.http.scaladsl.model.Uri import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileStorageMetadata -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.StorageTypeConfig import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageType} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.MoveFileRejection import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient @@ -28,14 +26,12 @@ object LinkFile { /** * Construct a [[LinkFile]] from the given ''storage''. */ - def apply(storage: Storage, client: RemoteDiskStorageClient, config: StorageTypeConfig)(implicit - as: ActorSystem, + def apply(storage: Storage, client: RemoteDiskStorageClient)(implicit uuidf: UUIDF ): LinkFile = storage match { - case storage: Storage.DiskStorage => unsupported(storage.tpe) - case storage: Storage.S3Storage => storage.linkFile(config) case storage: Storage.RemoteDiskStorage => storage.linkFile(client) + case _ => unsupported(storage.tpe) } private def unsupported(storageType: StorageType): LinkFile = diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3StorageLinkFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3StorageLinkFile.scala deleted file mode 100644 index bbcba24a30..0000000000 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3StorageLinkFile.scala +++ /dev/null @@ -1,70 +0,0 @@ -package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3 - -import akka.actor.ActorSystem -import akka.http.scaladsl.model.Uri -import akka.stream.alpakka.s3.scaladsl.S3 -import akka.stream.alpakka.s3.{S3Attributes, S3Exception} -import akka.stream.scaladsl.Sink -import cats.effect.IO -import cats.implicits._ -import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Storage -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileStorageMetadata -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.StorageTypeConfig -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.S3Storage -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.FetchFileRejection -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.{LinkFile, SaveFile} -import ch.epfl.bluebrain.nexus.delta.sdk.syntax._ - -import java.net.URLDecoder -import java.nio.charset.StandardCharsets.UTF_8 -import scala.concurrent.Future - -class S3StorageLinkFile(storage: S3Storage, config: StorageTypeConfig)(implicit as: ActorSystem, uuidf: UUIDF) - extends LinkFile { - - import as.dispatcher - - private val fileNotFoundException = new IllegalArgumentException("File not found") - - override def apply(key: Uri.Path, filename: String): IO[FileStorageMetadata] = { - - for { - uuid <- uuidf() - (contentLength, digest, location) <- storeFile(key) - } yield { - FileStorageMetadata( - uuid, - contentLength, - digest, - origin = Storage, - location, - key - ) - } - } - - private def storeFile(key: Uri.Path): IO[(Long, Digest.ComputedDigest, Uri)] = { - val attributes = S3Attributes.settings(storage.value.alpakkaSettings(config)) - val location: Uri = storage.value.address(storage.value.bucket) / key - IO.fromFuture( - IO.delay( - S3.download(storage.value.bucket, URLDecoder.decode(key.toString, UTF_8.toString)) - .withAttributes(attributes) - .runWith(Sink.head) - .flatMap { - case Some((source, meta)) => - source.runWith(SaveFile.digestSink(storage.value.algorithm)).map { dig => - (meta.contentLength, dig, location) - } - case None => Future.failed(fileNotFoundException) - } - ) - ).adaptError { - case `fileNotFoundException` => FetchFileRejection.FileNotFound(location.toString) - case err: S3Exception => FetchFileRejection.UnexpectedFetchError(key.toString, err.toString) - case err => FetchFileRejection.UnexpectedFetchError(key.toString, err.getMessage) - } - } -} diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/MinioSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/MinioSpec.scala index 2cba97dd14..515ff646ff 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/MinioSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/MinioSpec.scala @@ -15,8 +15,7 @@ import java.nio.charset.StandardCharsets.UTF_8 class MinioSpec extends Suites with MinioDocker { override val nestedSuites: IndexedSeq[Suite] = Vector( new S3StorageAccessSpec(this), - new S3StorageSaveAndFetchFileSpec(this), - new S3StorageLinkFileSpec(this) + new S3StorageSaveAndFetchFileSpec(this) ) } diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3StorageLinkFileSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3StorageLinkFileSpec.scala deleted file mode 100644 index 2c48f359d1..0000000000 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3StorageLinkFileSpec.scala +++ /dev/null @@ -1,101 +0,0 @@ -package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3 - -import akka.actor.ActorSystem -import akka.http.scaladsl.model.{HttpEntity, Uri} -import akka.testkit.TestKit -import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.ComputedDigest -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileStorageMetadata -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StorageFixtures -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.DigestAlgorithm -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.S3Storage -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageValue.S3StorageValue -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.AkkaSourceHelpers -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.FetchFileRejection.FileNotFound -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.MinioSpec._ -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.permissions.{read, write} -import ch.epfl.bluebrain.nexus.delta.sdk.syntax._ -import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef -import ch.epfl.bluebrain.nexus.testkit.minio.MinioDocker -import ch.epfl.bluebrain.nexus.testkit.minio.MinioDocker._ -import ch.epfl.bluebrain.nexus.testkit.scalatest.ce.CatsEffectSpec -import io.circe.Json -import org.scalatest.{BeforeAndAfterAll, DoNotDiscover} -import software.amazon.awssdk.regions.Region - -import java.util.UUID - -@DoNotDiscover -class S3StorageLinkFileSpec(docker: MinioDocker) - extends TestKit(ActorSystem("S3StorageSaveAndFetchFileSpec")) - with CatsEffectSpec - with AkkaSourceHelpers - with StorageFixtures - with BeforeAndAfterAll { - - private val iri = iri"http://localhost/s3" - private val uuid = UUID.fromString("8049ba90-7cc6-4de5-93a1-802c04200dcc") - implicit private val uuidf: UUIDF = UUIDF.fixed(uuid) - private val project = ProjectRef.unsafe("org", "project") - private val filename = "myfile.txt" - private val digest = { - ComputedDigest(DigestAlgorithm.default, "e0ac3601005dfa1864f5392aabaf7d898b1b5bab854f1acb4491bcd806b76b0c") - } - private val contentSize = 12L - - private val path = Uri.Path("org/project/8/0/4/9/b/a/9/0/myfile.txt") - private var location: String = _ - - private var storageValue: S3StorageValue = _ - private var storage: S3Storage = _ - - override protected def beforeAll(): Unit = { - location = s"http://bucket3.$VirtualHost:${docker.hostConfig.port}/org/project/8/0/4/9/b/a/9/0/myfile.txt" - storageValue = S3StorageValue( - default = false, - algorithm = DigestAlgorithm.default, - bucket = "bucket3", - endpoint = Some(docker.hostConfig.endpoint), - region = Some(Region.EU_CENTRAL_1), - readPermission = read, - writePermission = write, - maxFileSize = 20 - ) - createBucket(storageValue).accepted - storage = S3Storage(iri, project, storageValue, Json.obj()) - } - - override protected def afterAll(): Unit = - deleteBucket(storageValue).accepted - - "S3Storage linking operations" should { - val content = "file content" - val entity = HttpEntity(content) - - "succeed" in { - storage.saveFile(config).apply(filename, entity).accepted shouldEqual FileStorageMetadata( - uuid, - contentSize, - digest, - FileAttributesOrigin.Client, - location, - path - ) - - storage.linkFile(config).apply(path, filename).accepted shouldEqual FileStorageMetadata( - uuid, - contentSize, - digest, - FileAttributesOrigin.Storage, - location, - path - ) - } - - "fail linking a file that does not exist" in { - storage.linkFile(config).apply(Uri.Path("my/file-40.txt"), filename).rejectedWith[FileNotFound] - } - - } -} diff --git a/docs/src/main/paradox/docs/delta/api/files-api.md b/docs/src/main/paradox/docs/delta/api/files-api.md index a1132d44d4..425b110ce7 100644 --- a/docs/src/main/paradox/docs/delta/api/files-api.md +++ b/docs/src/main/paradox/docs/delta/api/files-api.md @@ -114,7 +114,7 @@ Response ## Link using POST -Brings a file existing in a storage to Nexus Delta as a file resource. This operation is supported for files using `S3Storage` and `RemoteDiskStorage`. +Brings a file existing in a storage to Nexus Delta as a file resource. This operation is supported for files using `RemoteDiskStorage`. ``` POST /v1/files/{org_label}/{project_label}?storage={storageId}&tag={tagName} @@ -150,7 +150,7 @@ Response ## Link using PUT -Brings a file existing in a storage to Nexus Delta as a file resource. This operation is supported for files using `S3Storage` and `RemoteDiskStorage`. +Brings a file existing in a storage to Nexus Delta as a file resource. This operation is supported for files using `RemoteDiskStorage`. This alternative endpoint allows to specify the resource `@id`. diff --git a/docs/src/main/paradox/docs/releases/v1.10-release-notes.md b/docs/src/main/paradox/docs/releases/v1.10-release-notes.md index d1383a779f..ce2f56e4d1 100644 --- a/docs/src/main/paradox/docs/releases/v1.10-release-notes.md +++ b/docs/src/main/paradox/docs/releases/v1.10-release-notes.md @@ -88,6 +88,7 @@ update operations. #### Deprecations * Storages can no longer be tagged, looked up by tag or have their tags fetched. +* S3 storages no longer support a linking operation. ### Resolvers diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/S3StorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/S3StorageSpec.scala index 39ca059ce1..34c970516d 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/S3StorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/S3StorageSpec.scala @@ -133,48 +133,16 @@ class S3StorageSpec extends StorageSpec { } s"Linking in S3" should { - "link an existing file" in { + "be rejected" in { val payload = Json.obj( "filename" -> Json.fromString("logo.png"), "path" -> Json.fromString(logoKey), "mediaType" -> Json.fromString("image/png") ) - val fileId = s"${config.deltaUri}/resources/$projectRef/_/logo.png" deltaClient.put[Json](s"/files/$projectRef/logo.png?storage=nxv:${storageId}2", payload, Coyote) { - (json, response) => - response.status shouldEqual StatusCodes.Created - filterMetadataKeys(json) shouldEqual - jsonContentOf( - "kg/files/linking-metadata.json", - replacements( - Coyote, - "projId" -> projectRef, - "self" -> fileSelf(projectRef, fileId), - "endpoint" -> s3Endpoint, - "endpointBucket" -> s3BucketEndpoint, - "key" -> logoKey - ): _* - ) + (_, response) => + response.status shouldEqual StatusCodes.BadRequest } } } - - "fail to link a nonexistent file" in { - val payload = Json.obj( - "filename" -> Json.fromString("logo.png"), - "path" -> Json.fromString("non/existent.png"), - "mediaType" -> Json.fromString("image/png") - ) - - deltaClient.put[Json](s"/files/$projectRef/nonexistent.png?storage=nxv:${storageId}2", payload, Coyote) { - (json, response) => - response.status shouldEqual StatusCodes.BadRequest - json shouldEqual jsonContentOf( - "kg/files/linking-notfound.json", - "org" -> orgId, - "proj" -> projId, - "endpointBucket" -> s3BucketEndpoint - ) - } - } }