diff --git a/delta/plugins/storage/src/main/resources/storage.conf b/delta/plugins/storage/src/main/resources/storage.conf index dbf3d4d28e..1a42e74250 100644 --- a/delta/plugins/storage/src/main/resources/storage.conf +++ b/delta/plugins/storage/src/main/resources/storage.conf @@ -21,9 +21,7 @@ plugins.storage { default-write-permission = "files/write" # flag to decide whether or not to show the absolute location of the files in the metadata response show-location = false - # the default capacity for storages (in bytes), by default no limit is defined - default-capacity = null - # the default maximum allowed file size (in bytes) for uploaded files. 10 GB + # the default maximum allowed file size (in bytes) for uploaded files. 10 GB default-max-file-size = 10737418240 } # S3 compatible storage configuration diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala index 7026d6c1a0..9e61ca4d11 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala @@ -218,12 +218,11 @@ class StoragePluginModule(priority: Int) extends ModuleDef { files: Files, storages: Storages, aclCheck: AclCheck, - storagesStatistics: StoragesStatistics, diskCopy: DiskStorageCopyFiles, remoteDiskCopy: RemoteDiskStorageCopyFiles, uuidF: UUIDF ) => - BatchCopy.mk(files, storages, aclCheck, storagesStatistics, diskCopy, remoteDiskCopy)(uuidF) + BatchCopy.mk(files, storages, aclCheck, diskCopy, remoteDiskCopy)(uuidF) } make[BatchFiles].from { diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StorageScopeInitialization.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StorageScopeInitialization.scala index e58c7a67db..a53c1644b6 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StorageScopeInitialization.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StorageScopeInitialization.scala @@ -74,7 +74,6 @@ object StorageScopeInitialization { volume = None, readPermission = None, writePermission = None, - capacity = None, maxFileSize = None ) new StorageScopeInitialization(storages, serviceAccount, defaultStorageId, defaultFields) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala index 8159dc7e6b..249719dcb2 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala @@ -7,6 +7,7 @@ import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.FetchFileResource import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.FetchStorage import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.{DiskStorage, RemoteDiskStorage} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageType} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection @@ -14,8 +15,6 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.Storage import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{DiskCopyDetails, DiskStorageCopyFiles} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.RemoteDiskStorageCopyFiles import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.RemoteDiskCopyDetails -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{FetchStorage, StoragesStatistics} -import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck import ch.epfl.bluebrain.nexus.delta.sdk.error.ServiceError.AuthorizationFailed import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller @@ -32,7 +31,6 @@ object BatchCopy { fetchFile: FetchFileResource, fetchStorage: FetchStorage, aclCheck: AclCheck, - storagesStatistics: StoragesStatistics, diskCopy: DiskStorageCopyFiles, remoteDiskCopy: RemoteDiskStorageCopyFiles )(implicit uuidF: UUIDF): BatchCopy = new BatchCopy { @@ -68,21 +66,9 @@ object BatchCopy { private def validateFilesForStorage(destStorage: Storage, sourcesBytes: NonEmptyList[Long]): IO[Unit] = { val maxSize = destStorage.storageValue.maxFileSize - for { - _ <- IO.raiseWhen(sourcesBytes.exists(_ > maxSize))(SourceFileTooLarge(maxSize, destStorage.id)) - _ <- validateRemainingStorageCapacity(destStorage, sourcesBytes) - } yield () + IO.raiseWhen(sourcesBytes.exists(_ > maxSize))(SourceFileTooLarge(maxSize, destStorage.id)) } - private def validateRemainingStorageCapacity(destStorage: Storage, sourcesBytes: NonEmptyList[Long]) = - for { - spaceLeft <- storagesStatistics.getStorageAvailableSpace(destStorage) - totalSize = sourcesBytes.toList.sum - _ <- spaceLeft - .collectFirst { case s if s < totalSize => notEnoughSpace(totalSize, s, destStorage.id) } - .getOrElse(IO.unit) - } yield () - private def fetchDiskCopyDetails(destStorage: DiskStorage, fileId: FileId)(implicit c: Caller ): IO[DiskCopyDetails] = @@ -122,9 +108,6 @@ object BatchCopy { private def unsupported(tpe: StorageType) = IO.raiseError(CopyFileRejection.UnsupportedOperation(tpe)) - private def notEnoughSpace(totalSize: Long, spaceLeft: Long, destStorage: Iri) = - IO.raiseError(TotalCopySizeTooLarge(totalSize, spaceLeft, destStorage)) - private def fetchFileAndValidateStorage(id: FileId)(implicit c: Caller) = { for { file <- fetchFile.fetch(id) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesConfig.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesConfig.scala index d863bfe5dc..653d880391 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesConfig.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesConfig.scala @@ -134,8 +134,6 @@ object StoragesConfig { * the default permission required in order to upload a file to a disk storage * @param showLocation * flag to decide whether or not to show the absolute location of the files in the metadata response - * @param defaultCapacity - * the default capacity available to store the files * @param defaultMaxFileSize * the default maximum allowed file size (in bytes) for uploaded files */ @@ -146,7 +144,6 @@ object StoragesConfig { defaultReadPermission: Permission, defaultWritePermission: Permission, showLocation: Boolean, - defaultCapacity: Option[Long], defaultMaxFileSize: Long ) extends StorageTypeEntryConfig diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesStatistics.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesStatistics.scala index 84f11822dd..e047e2c92e 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesStatistics.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesStatistics.scala @@ -4,7 +4,7 @@ import akka.http.scaladsl.model.Uri.Query import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.EventMetricsProjection.eventMetricsIndex import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.ElasticSearchClient -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageStatEntry} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageStatEntry import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegment import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef @@ -18,17 +18,6 @@ trait StoragesStatistics { */ def get(idSegment: IdSegment, project: ProjectRef): IO[StorageStatEntry] - /** - * Retrieve remaining space on a storage if it has a capacity. - */ - final def getStorageAvailableSpace(storage: Storage): IO[Option[Long]] = - storage.storageValue.capacity.fold(IO.none[Long]) { capacity => - get(storage.id, storage.project) - .redeem( - _ => Some(capacity), - stat => Some(capacity - stat.spaceUsed) - ) - } } object StoragesStatistics { diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageFields.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageFields.scala index 41f907dbf9..3c94ce2d6a 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageFields.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageFields.scala @@ -87,8 +87,6 @@ object StorageFields { * the permission required in order to download a file from this storage * @param writePermission * the permission required in order to upload a file to this storage - * @param capacity - * the capacity available (in bytes) to store files * @param maxFileSize * the maximum allowed file size (in bytes) for uploaded files */ @@ -99,7 +97,6 @@ object StorageFields { volume: Option[AbsolutePath], readPermission: Option[Permission], writePermission: Option[Permission], - capacity: Option[Long], maxFileSize: Option[Long] ) extends StorageFields { override val tpe: StorageType = StorageType.DiskStorage @@ -116,7 +113,6 @@ object StorageFields { volume.getOrElse(config.disk.defaultVolume), readPermission.getOrElse(config.disk.defaultReadPermission), writePermission.getOrElse(config.disk.defaultWritePermission), - capacity.orElse(config.disk.defaultCapacity), computeMaxFileSize(maxFileSize, config.disk.defaultMaxFileSize) ) ) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageValue.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageValue.scala index 35ba507013..1e921f8284 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageValue.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageValue.scala @@ -48,12 +48,6 @@ sealed trait StorageValue extends Product with Serializable { */ def algorithm: DigestAlgorithm - /** - * @return - * the maximum allocated capacity for the storage - */ - def capacity: Option[Long] - /** * @return * the maximum allowed file size (in bytes) for uploaded files @@ -89,7 +83,6 @@ object StorageValue { volume: AbsolutePath, readPermission: Permission, writePermission: Permission, - capacity: Option[Long], maxFileSize: Long ) extends StorageValue { @@ -111,10 +104,9 @@ object StorageValue { volume: AbsolutePath, readPermission: Permission, writePermission: Permission, - capacity: Option[Long], maxFileSize: Long ): DiskStorageValue = - DiskStorageValue(None, None, default, algorithm, volume, readPermission, writePermission, capacity, maxFileSize) + DiskStorageValue(None, None, default, algorithm, volume, readPermission, writePermission, maxFileSize) } @@ -135,8 +127,8 @@ object StorageValue { maxFileSize: Long ) extends StorageValue { - override val tpe: StorageType = StorageType.S3Storage - override val capacity: Option[Long] = None + override val tpe: StorageType = StorageType.S3Storage + } object S3StorageValue { @@ -182,8 +174,8 @@ object StorageValue { maxFileSize: Long ) extends StorageValue { - override val tpe: StorageType = StorageType.RemoteDiskStorage - override val capacity: Option[Long] = None + override val tpe: StorageType = StorageType.RemoteDiskStorage + } object RemoteDiskStorageValue { diff --git a/delta/plugins/storage/src/test/resources/storages/database/disk-storage-created.json b/delta/plugins/storage/src/test/resources/storages/database/disk-storage-created.json index 5e6e461f08..e5079f508d 100644 --- a/delta/plugins/storage/src/test/resources/storages/database/disk-storage-created.json +++ b/delta/plugins/storage/src/test/resources/storages/database/disk-storage-created.json @@ -9,7 +9,6 @@ "volume": "/tmp", "readPermission": "disk/read", "writePermission": "disk/write", - "capacity" : 1000, "maxFileSize": 50, "@type": "DiskStorageValue" }, @@ -21,7 +20,6 @@ "default": true, "readPermission": "disk/read", "writePermission": "disk/write", - "capacity" : 1000, "maxFileSize": 50 }, "rev": 1, diff --git a/delta/plugins/storage/src/test/resources/storages/database/disk-storage-updated.json b/delta/plugins/storage/src/test/resources/storages/database/disk-storage-updated.json index 4308709f7f..316d9a7722 100644 --- a/delta/plugins/storage/src/test/resources/storages/database/disk-storage-updated.json +++ b/delta/plugins/storage/src/test/resources/storages/database/disk-storage-updated.json @@ -9,7 +9,6 @@ "volume" : "/tmp", "readPermission" : "disk/read", "writePermission" : "disk/write", - "capacity" : 2000, "maxFileSize" : 40, "@type" : "DiskStorageValue" }, @@ -21,7 +20,6 @@ "default" : true, "readPermission" : "disk/read", "writePermission" : "disk/write", - "capacity" : 1000, "maxFileSize" : 50 }, "rev" : 2, diff --git a/delta/plugins/storage/src/test/resources/storages/disk-storage-expanded.json b/delta/plugins/storage/src/test/resources/storages/disk-storage-expanded.json index 875239d721..e4b6ba0083 100644 --- a/delta/plugins/storage/src/test/resources/storages/disk-storage-expanded.json +++ b/delta/plugins/storage/src/test/resources/storages/disk-storage-expanded.json @@ -25,11 +25,6 @@ "@value": true } ], - "https://bluebrain.github.io/nexus/vocabulary/capacity": [ - { - "@value": 1000 - } - ], "https://bluebrain.github.io/nexus/vocabulary/maxFileSize": [ { "@value": 50 diff --git a/delta/plugins/storage/src/test/resources/storages/disk-storage.json b/delta/plugins/storage/src/test/resources/storages/disk-storage.json index fe030e19c8..38dfae14b8 100644 --- a/delta/plugins/storage/src/test/resources/storages/disk-storage.json +++ b/delta/plugins/storage/src/test/resources/storages/disk-storage.json @@ -9,6 +9,5 @@ "default": true, "readPermission": "disk/read", "writePermission": "disk/write", - "capacity": 1000, "maxFileSize": 50 } \ No newline at end of file diff --git a/delta/plugins/storage/src/test/resources/storages/sse/disk-storage-created.json b/delta/plugins/storage/src/test/resources/storages/sse/disk-storage-created.json index 8e9562b7c0..7bb94f29a7 100644 --- a/delta/plugins/storage/src/test/resources/storages/sse/disk-storage-created.json +++ b/delta/plugins/storage/src/test/resources/storages/sse/disk-storage-created.json @@ -13,7 +13,6 @@ "@type": "DiskStorage", "name": "diskName", "description": "diskDescription", - "capacity": 1000, "default": true, "maxFileSize": 50, "readPermission": "disk/read", diff --git a/delta/plugins/storage/src/test/resources/storages/sse/disk-storage-updated.json b/delta/plugins/storage/src/test/resources/storages/sse/disk-storage-updated.json index 981b32976b..1ce1179ee2 100644 --- a/delta/plugins/storage/src/test/resources/storages/sse/disk-storage-updated.json +++ b/delta/plugins/storage/src/test/resources/storages/sse/disk-storage-updated.json @@ -13,7 +13,6 @@ "@type": "DiskStorage", "name": "diskName", "description": "diskDescription", - "capacity": 1000, "default": true, "maxFileSize": 50, "readPermission": "disk/read", diff --git a/delta/plugins/storage/src/test/resources/storages/storage-disk-state.json b/delta/plugins/storage/src/test/resources/storages/storage-disk-state.json index b373bc4118..c10b1b1a12 100644 --- a/delta/plugins/storage/src/test/resources/storages/storage-disk-state.json +++ b/delta/plugins/storage/src/test/resources/storages/storage-disk-state.json @@ -15,7 +15,6 @@ "volume": "/tmp", "readPermission": "disk/read", "writePermission": "disk/write", - "capacity": 1000, "maxFileSize": 50, "@type": "DiskStorageValue" }, @@ -27,7 +26,6 @@ "default": true, "readPermission": "disk/read", "writePermission": "disk/write", - "capacity": 1000, "maxFileSize": 50 }, "createdAt": "1970-01-01T00:00:00Z", diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StorageScopeInitializationSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StorageScopeInitializationSpec.scala index f6b043eeff..733a3f27fc 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StorageScopeInitializationSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StorageScopeInitializationSpec.scala @@ -64,7 +64,6 @@ class StorageScopeInitializationSpec volume = config.disk.defaultVolume, readPermission = config.disk.defaultReadPermission, writePermission = config.disk.defaultWritePermission, - capacity = config.disk.defaultCapacity, maxFileSize = config.disk.defaultMaxFileSize ) resource.rev shouldEqual 1L diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala index 2d55df362e..88a20976c9 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala @@ -167,7 +167,7 @@ class FilesSpec(fixture: RemoteStorageClientFixtures) "creating a file" should { "create storages for files" in { - val payload = diskFieldsJson deepMerge json"""{"capacity": 320, "maxFileSize": 300, "volume": "$path"}""" + val payload = diskFieldsJson deepMerge json"""{"maxFileSize": 300, "volume": "$path"}""" storages.create(diskId, projectRef, payload).accepted val payload2 = diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopySuite.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopySuite.scala index 6f6d85734d..8c30559984 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopySuite.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopySuite.scala @@ -5,14 +5,13 @@ import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.BatchCopySuite._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks._ -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileDescription, FileMetadata} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileId, LimitedFileAttributes} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FetchFileResource, FileFixtures, FileResource} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.{DiskStorage, RemoteDiskStorage, S3Storage} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageStatEntry, StorageType} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection.{DifferentStorageTypes, SourceFileTooLarge, TotalCopySizeTooLarge, UnsupportedOperation} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageType} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection.{DifferentStorageTypes, SourceFileTooLarge, UnsupportedOperation} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{DiskCopyDetails, DiskStorageCopyFiles} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.RemoteDiskStorageCopyFiles import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.RemoteDiskCopyDetails @@ -21,7 +20,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.acls.model.AclAddress import ch.epfl.bluebrain.nexus.delta.sdk.acls.{AclCheck, AclSimpleCheck} import ch.epfl.bluebrain.nexus.delta.sdk.error.ServiceError.AuthorizationFailed import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller -import ch.epfl.bluebrain.nexus.delta.sdk.model.{IdSegment, IdSegmentRef} +import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegmentRef import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.User import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef} import ch.epfl.bluebrain.nexus.testkit.Generators @@ -31,14 +30,13 @@ import scala.collection.mutable.ListBuffer class BatchCopySuite extends NexusSuite with StorageFixtures with Generators with FileFixtures with FileGen { - private val sourceProj = genProject() - private val sourceFileId = genFileId(sourceProj.ref) - private val source = CopyFileSource(sourceProj.ref, NonEmptyList.of(sourceFileId)) - private val storageStatEntry = StorageStatEntry(files = 10L, spaceUsed = 5L) - private val keywords = genKeywords() - private val description = genString() - private val name = genString() - private val stubbedFileAttr = + private val sourceProj = genProject() + private val sourceFileId = genFileId(sourceProj.ref) + private val source = CopyFileSource(sourceProj.ref, NonEmptyList.of(sourceFileId)) + private val keywords = genKeywords() + private val description = genString() + private val name = genString() + private val stubbedFileAttr = attributes(genString(), keywords = keywords, description = Some(description), name = Some(name)) test("successfully perform disk copy") { @@ -51,7 +49,6 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit fetchFile = stubbedFetchFile(sourceFileRes, events), fetchStorage = stubbedFetchStorage(sourceStorage, events), aclCheck = aclCheck, - stats = stubbedStorageStats(storageStatEntry, events), diskCopy = stubbedDiskCopy(NonEmptyList.of(stubbedFileAttr), events) ) val destStorage: DiskStorage = genDiskStorage() @@ -61,7 +58,6 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit assertEquals(obtained, NonEmptyList.of(stubbedFileAttr)) sourceFileWasFetched(obtainedEvents, sourceFileId) sourceStorageWasFetched(obtainedEvents, sourceFileRes.value.storage, sourceProj.ref) - destinationDiskStorageStatsWereFetched(obtainedEvents, destStorage) diskCopyWasPerformed( obtainedEvents, destStorage, @@ -80,7 +76,6 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit fetchFile = stubbedFetchFile(sourceFileRes, events), fetchStorage = stubbedFetchStorage(sourceStorage, events), aclCheck = aclCheck, - stats = stubbedStorageStats(storageStatEntry, events), remoteCopy = stubbedRemoteCopy(NonEmptyList.of(stubbedFileAttr), events) ) val destStorage: RemoteDiskStorage = genRemoteStorage() @@ -90,7 +85,6 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit assertEquals(obtained, NonEmptyList.of(stubbedFileAttr)) sourceFileWasFetched(obtainedEvents, sourceFileId) sourceStorageWasFetched(obtainedEvents, sourceFileRes.value.storage, sourceProj.ref) - destinationRemoteStorageStatsWereNotFetched(obtainedEvents) remoteDiskCopyWasPerformed( obtainedEvents, destStorage, @@ -167,43 +161,13 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit } } - test("fail if total size of source files is too large for destination disk storage") { - val events = ListBuffer.empty[Event] - val fileSize = 6L - val capacity = 10L - val statEntry = StorageStatEntry(files = 10L, spaceUsed = 1L) - val spaceLeft = capacity - statEntry.spaceUsed - val (sourceFileRes, sourceStorage) = - genFileResourceAndStorage(sourceFileId, sourceProj.context, diskVal, keywords, description, name, fileSize) - val (user, aclCheck) = userAuthorizedOnProjectStorage(sourceStorage.value) - - val batchCopy = mkBatchCopy( - fetchFile = stubbedFetchFile(sourceFileRes, events), - fetchStorage = stubbedFetchStorage(sourceStorage, events), - aclCheck = aclCheck, - stats = stubbedStorageStats(statEntry, events) - ) - - val destStorage = genDiskStorageWithCapacity(capacity) - val error = TotalCopySizeTooLarge(fileSize * 2, spaceLeft, destStorage.id) - val twoFileSource = CopyFileSource(sourceProj.ref, NonEmptyList.of(sourceFileId, sourceFileId)) - - batchCopy.copyFiles(twoFileSource, destStorage)(caller(user)).interceptEquals(error).map { _ => - val obtainedEvents = events.toList - sourceFileWasFetched(obtainedEvents, sourceFileId) - sourceStorageWasFetched(obtainedEvents, sourceFileRes.value.storage, sourceProj.ref) - destinationDiskStorageStatsWereFetched(obtainedEvents, destStorage) - } - } - private def mkBatchCopy( fetchFile: FetchFileResource = FetchFileResourceMock.unimplemented, fetchStorage: FetchStorage = FetchStorageMock.unimplemented, aclCheck: AclCheck = AclSimpleCheck().accepted, - stats: StoragesStatistics = StoragesStatisticsMock.unimplemented, diskCopy: DiskStorageCopyFiles = DiskCopyMock.unimplemented, remoteCopy: RemoteDiskStorageCopyFiles = RemoteCopyMock.unimplemented - ): BatchCopy = BatchCopy.mk(fetchFile, fetchStorage, aclCheck, stats, diskCopy, remoteCopy) + ): BatchCopy = BatchCopy.mk(fetchFile, fetchStorage, aclCheck, diskCopy, remoteCopy) private def userAuthorizedOnProjectStorage(storage: Storage): (User, AclCheck) = { val user = genUser() @@ -211,31 +175,21 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit (user, AclSimpleCheck((user, AclAddress.fromProject(storage.project), permissions)).accepted) } - private def sourceFileWasFetched(events: List[Event], id: FileId) = { + private def sourceFileWasFetched(events: List[Event], id: FileId): Unit = { val obtained = events.collectFirst { case f: FetchFileCalled => f } assertEquals(obtained, Some(FetchFileCalled(id))) } - private def sourceStorageWasFetched(events: List[Event], storageRef: ResourceRef.Revision, proj: ProjectRef) = { + private def sourceStorageWasFetched(events: List[Event], storageRef: ResourceRef.Revision, proj: ProjectRef): Unit = { val obtained = events.collectFirst { case f: FetchStorageCalled => f } assertEquals(obtained, Some(FetchStorageCalled(IdSegmentRef(storageRef), proj))) } - private def destinationDiskStorageStatsWereFetched(events: List[Event], storage: DiskStorage) = { - val obtained = events.collectFirst { case f: StoragesStatsCalled => f } - assertEquals(obtained, Some(StoragesStatsCalled(storage.id, storage.project))) - } - - private def destinationRemoteStorageStatsWereNotFetched(events: List[Event]) = { - val obtained = events.collectFirst { case f: StoragesStatsCalled => f } - assertEquals(obtained, None) - } - private def diskCopyWasPerformed( events: List[Event], storage: DiskStorage, sourceAttr: LimitedFileAttributes - ) = { + ): Unit = { val expectedDiskCopyDetails = DiskCopyDetails(storage, sourceAttr) val obtained = events.collectFirst { case f: DiskCopyCalled => f } assertEquals(obtained, Some(DiskCopyCalled(storage, NonEmptyList.of(expectedDiskCopyDetails)))) @@ -245,7 +199,7 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit events: List[Event], storage: RemoteDiskStorage, sourceAttr: FileAttributes - ) = { + ): Unit = { val expectedCopyDetails = RemoteDiskCopyDetails( uuid, @@ -261,12 +215,8 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit private def caller(user: User): Caller = Caller(user, Set(user)) - private def genDiskStorageWithCapacity(capacity: Long) = { - val limitedDiskVal = diskVal.copy(capacity = Some(capacity)) - DiskStorage(nxv + genString(), genProject().ref, limitedDiskVal, json"""{"disk": "value"}""") - } - - private def genDiskStorage() = genDiskStorageWithCapacity(1000L) + private def genDiskStorage() = + DiskStorage(nxv + genString(), genProject().ref, diskVal, json"""{"disk": "value"}""") private def genRemoteStorage() = RemoteDiskStorage(nxv + genString(), genProject().ref, remoteVal, json"""{"disk": "value"}""") @@ -277,20 +227,14 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit object BatchCopySuite { sealed trait Event - final case class FetchFileCalled(id: FileId) extends Event - final case class FetchStorageCalled(id: IdSegmentRef, project: ProjectRef) extends Event - final case class StoragesStatsCalled(idSegment: IdSegment, project: ProjectRef) extends Event + final case class FetchFileCalled(id: FileId) extends Event + final case class FetchStorageCalled(id: IdSegmentRef, project: ProjectRef) extends Event final case class DiskCopyCalled(destStorage: Storage.DiskStorage, details: NonEmptyList[DiskCopyDetails]) extends Event final case class RemoteCopyCalled( destStorage: Storage.RemoteDiskStorage, copyDetails: NonEmptyList[RemoteDiskCopyDetails] - ) extends Event - - private def stubbedStorageStats(storageStatEntry: StorageStatEntry, events: ListBuffer[Event]) = - StoragesStatisticsMock.withMockedGet((id, proj) => - addEventAndReturn(events, StoragesStatsCalled(id, proj), storageStatEntry) - ) + ) extends Event private def stubbedFetchFile(sourceFileRes: FileResource, events: ListBuffer[Event]) = FetchFileResourceMock.withMockedFetch(id => addEventAndReturn(events, FetchFileCalled(id), sourceFileRes)) diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala index a45c2a1bf4..7b68f35940 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala @@ -12,10 +12,10 @@ import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks.FileOperationsMock import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.ComputedDigest import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileId} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{contexts => fileContexts, permissions, FileFixtures, Files, FilesConfig} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FileFixtures, Files, FilesConfig, permissions, contexts => fileContexts} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageType import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.FileOperations -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{contexts => storageContexts, permissions => storagesPermissions, StorageFixtures, Storages, StoragesConfig} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{StorageFixtures, Storages, StoragesConfig, contexts => storageContexts, permissions => storagesPermissions} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.RdfMediaTypes.`application/ld+json` import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary @@ -154,7 +154,7 @@ class FilesRoutesSpec val s3Perms = json"""{"readPermission": "$s3Read", "writePermission": "$s3Write"}""" storages.create(s3Id, projectRef, diskFieldsJson deepMerge defaults deepMerge s3Perms)(callerWriter).accepted storages - .create(dId, projectRef, diskFieldsJson deepMerge defaults deepMerge json"""{"capacity":5000}""")(callerWriter) + .create(dId, projectRef, diskFieldsJson deepMerge defaults)(callerWriter) .void .accepted } diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StorageFixtures.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StorageFixtures.scala index ba4718230a..d6eb41e1aa 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StorageFixtures.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StorageFixtures.scala @@ -31,15 +31,15 @@ trait StorageFixtures extends CirceLiteral { // format: off implicit val config: StorageTypeConfig = StorageTypeConfig( - disk = DiskStorageConfig(diskVolume, Set(diskVolume,tmpVolume), DigestAlgorithm.default, permissions.read, permissions.write, showLocation = false, Some(5000), 50), + disk = DiskStorageConfig(diskVolume, Set(diskVolume,tmpVolume), DigestAlgorithm.default, permissions.read, permissions.write, showLocation = false, 50), amazon = Some(S3StorageConfig(DigestAlgorithm.default, "localhost", Secret(MinioDocker.RootUser), Secret(MinioDocker.RootPassword), permissions.read, permissions.write, showLocation = false, 60)), remoteDisk = Some(RemoteDiskStorageConfig(DigestAlgorithm.default, BaseUri("http://localhost", Label.unsafe("v1")), Anonymous, permissions.read, permissions.write, showLocation = false, 70, 50.millis)), ) implicit val showLocation: StoragesConfig.ShowFileLocation = config.showFileLocation - val diskFields = DiskStorageFields(Some("diskName"), Some("diskDescription"), default = true, Some(tmpVolume), Some(Permission.unsafe("disk/read")), Some(Permission.unsafe("disk/write")), Some(1000), Some(50)) + val diskFields = DiskStorageFields(Some("diskName"), Some("diskDescription"), default = true, Some(tmpVolume), Some(Permission.unsafe("disk/read")), Some(Permission.unsafe("disk/write")), Some(50)) val diskVal = diskFields.toValue(config).get - val diskFieldsUpdate = DiskStorageFields(Some("diskName"), Some("diskDescription"), default = false, Some(tmpVolume), Some(Permission.unsafe("disk/read")), Some(Permission.unsafe("disk/write")), Some(2000), Some(40)) + val diskFieldsUpdate = DiskStorageFields(Some("diskName"), Some("diskDescription"), default = false, Some(tmpVolume), Some(Permission.unsafe("disk/read")), Some(Permission.unsafe("disk/write")), Some(40)) val diskValUpdate = diskFieldsUpdate.toValue(config).get val s3Fields = S3StorageFields(Some("s3name"), Some("s3description"), default = true, "mybucket", Some(Permission.unsafe("s3/read")), Some(Permission.unsafe("s3/write")), Some(51)) val s3Val = s3Fields.toValue(config).get diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesSpec.scala index b00ff1787b..adea8ba835 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesSpec.scala @@ -109,7 +109,7 @@ private class StoragesSpec "updating a storage" should { "succeed" in { - val payload = diskFieldsJson deepMerge json"""{"default": false, "capacity": 2000, "maxFileSize": 40}""" + val payload = diskFieldsJson deepMerge json"""{"default": false, "maxFileSize": 40}""" storages.update(dId, projectRef, 2, payload).accepted shouldEqual resourceFor(dId, projectRef, diskValUpdate, payload, rev = 3, createdBy = bob, updatedBy = bob) } diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesStmSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesStmSpec.scala index 8234dc148d..6141c05f9e 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesStmSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesStmSpec.scala @@ -248,7 +248,7 @@ class StoragesStmSpec extends CatsEffectSpec with StorageFixtures { val diskVolume = AbsolutePath(Files.createTempDirectory("disk")).rightValue // format: off val config: StorageTypeConfig = StorageTypeConfig( - disk = DiskStorageConfig(diskVolume, Set(diskVolume), DigestAlgorithm.default, permissions.read, permissions.write, showLocation = false, Some(1000), 150), + disk = DiskStorageConfig(diskVolume, Set(diskVolume), DigestAlgorithm.default, permissions.read, permissions.write, showLocation = false, 150), amazon = None, remoteDisk = None ) diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageFieldsSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageFieldsSpec.scala index d7fc6c8fa0..af22e95e5a 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageFieldsSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageFieldsSpec.scala @@ -33,12 +33,11 @@ class StorageFieldsSpec extends CatsEffectSpec with RemoteContextResolutionFixtu "description", "readPermission", "writePermission", - "capacity", "maxFileSize", "volume" ) sourceDecoder(pc, jsonNoDefaults).accepted._2 shouldEqual - DiskStorageFields(None, None, default = true, None, None, None, None, None) + DiskStorageFields(None, None, default = true, None, None, None, None) } } diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFileSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFileSpec.scala index 72ea35a37f..10032ccc7d 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFileSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFileSpec.scala @@ -37,7 +37,7 @@ class DiskStorageSaveFileSpec "A DiskStorage saving operations" should { val iri = iri"http://localhost/disk" val project = ProjectRef.unsafe("org", "project") - val value = DiskStorageValue(default = true, DigestAlgorithm.default, volume, read, write, Some(100), 10) + val value = DiskStorageValue(default = true, DigestAlgorithm.default, volume, read, write, 10) val storage = DiskStorage(iri, project, value, Json.obj()) val content = "file content" val entity = HttpEntity(content) diff --git a/docs/src/main/paradox/docs/delta/api/storages-api.md b/docs/src/main/paradox/docs/delta/api/storages-api.md index 3412ef098a..ffd63875d5 100644 --- a/docs/src/main/paradox/docs/delta/api/storages-api.md +++ b/docs/src/main/paradox/docs/delta/api/storages-api.md @@ -36,7 +36,6 @@ local file-system structure and that Nexus has read and write access to the targ "volume": "{volume}", "readPermission": "{read_permission}", "writePermission": "{write_permission}", - "capacity": "{capacity}", "maxFileSize": {max_file_size} } ``` @@ -47,7 +46,6 @@ local file-system structure and that Nexus has read and write access to the targ - `{volume}`: String - the path to the local file-system volume where files using this storage will be stored. This field is optional, defaulting to the configuration flag `plugins.storage.storages.disk.default-volume` (`/tmp`). - `{read_permission}`: String - the permission a client must have in order to fetch files using this storage. This field is optional, defaulting to the configuration flag `plugins.storage.storages.disk.default-read-permission` (`resources/read`). - `{write_permission}`: String - the permission a client must have in order to create files using this storage. This field is optional, defaulting to the configuration flag `plugins.storage.storages.disk.default-write-permission` (`files/write`). -- `{capacity}`: Long - the maximum allocated capacity in bytes for storing files using this storage. This field is optional, defaulting to the configuration flag `plugins.storage.storages.disk.default-capacity` (None). - `{max_file_size}`: Long - the maximum allowed size in bytes for files uploaded using this storage. This field is optional, defaulting to the configuration flag `plugins.storage.storages.disk.default-max-file-size` (10G). ### Remote disk storage diff --git a/docs/src/main/paradox/docs/releases/v1.10-release-notes.md b/docs/src/main/paradox/docs/releases/v1.10-release-notes.md index ce2f56e4d1..de1492a2e4 100644 --- a/docs/src/main/paradox/docs/releases/v1.10-release-notes.md +++ b/docs/src/main/paradox/docs/releases/v1.10-release-notes.md @@ -80,7 +80,7 @@ Previously deprecated storages can now be undeprecated. @ref:[More information](../delta/api/storages-api.md#undeprecate) -#### The endpoint for remote storages can not be overriden anymore +#### The endpoint for remote storages can not be overridden anymore The endpoint for remote storages is now only defined by configuration and can not be overridden in the create and update operations. @@ -89,6 +89,7 @@ update operations. * Storages can no longer be tagged, looked up by tag or have their tags fetched. * S3 storages no longer support a linking operation. +* It is no longer possible to provide the storage capacity as it is no longer enforced. ### Resolvers diff --git a/ship/src/main/resources/ship-default.conf b/ship/src/main/resources/ship-default.conf index f5759a1cab..2c6404e7b6 100644 --- a/ship/src/main/resources/ship-default.conf +++ b/ship/src/main/resources/ship-default.conf @@ -115,8 +115,6 @@ ship { default-write-permission = "files/write" # flag to decide whether or not to show the absolute location of the files in the metadata response show-location = false - # the default capacity for storages (in bytes), by default no limit is defined - default-capacity = null # the default maximum allowed file size (in bytes) for uploaded files. 10 GB default-max-file-size = 10737418240 }