Remove capacity

BlueBrain · Apr 18, 2024 · a4de31a · a4de31a
1 parent b16f700
commit a4de31a
Show file tree

Hide file tree

Showing 27 changed files with 44 additions and 166 deletions.
diff --git a/delta/plugins/storage/src/main/resources/storage.conf b/delta/plugins/storage/src/main/resources/storage.conf
@@ -21,9 +21,7 @@ plugins.storage {
       default-write-permission = "files/write"
       # flag to decide whether or not to show the absolute location of the files in the metadata response
       show-location = false
-      # the default capacity for storages (in bytes), by default no limit is defined
-      default-capacity = null
-      # the default maximum allowed file size (in bytes) for uploaded files. 10 GB 
+      # the default maximum allowed file size (in bytes) for uploaded files. 10 GB
       default-max-file-size = 10737418240
     }
     # S3 compatible storage configuration

diff --git a/...ge/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala b/...ge/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala
@@ -218,12 +218,11 @@ class StoragePluginModule(priority: Int) extends ModuleDef {
         files: Files,
         storages: Storages,
         aclCheck: AclCheck,
-        storagesStatistics: StoragesStatistics,
         diskCopy: DiskStorageCopyFiles,
         remoteDiskCopy: RemoteDiskStorageCopyFiles,
         uuidF: UUIDF
     ) =>
-      BatchCopy.mk(files, storages, aclCheck, storagesStatistics, diskCopy, remoteDiskCopy)(uuidF)
+      BatchCopy.mk(files, storages, aclCheck, diskCopy, remoteDiskCopy)(uuidF)
   }
 
   make[BatchFiles].from {

diff --git a/...main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StorageScopeInitialization.scala b/...main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StorageScopeInitialization.scala
@@ -74,7 +74,6 @@ object StorageScopeInitialization {
       volume = None,
       readPermission = None,
       writePermission = None,
-      capacity = None,
       maxFileSize = None
     )
     new StorageScopeInitialization(storages, serviceAccount, defaultStorageId, defaultFields)

diff --git a/.../src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala b/.../src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala
@@ -7,15 +7,14 @@ import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.FetchFileResource
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model._
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource
+import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.FetchStorage
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.{DiskStorage, RemoteDiskStorage}
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageType}
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection._
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{DiskCopyDetails, DiskStorageCopyFiles}
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.RemoteDiskStorageCopyFiles
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.RemoteDiskCopyDetails
-import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{FetchStorage, StoragesStatistics}
-import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri
 import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck
 import ch.epfl.bluebrain.nexus.delta.sdk.error.ServiceError.AuthorizationFailed
 import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller
@@ -32,7 +31,6 @@ object BatchCopy {
       fetchFile: FetchFileResource,
       fetchStorage: FetchStorage,
       aclCheck: AclCheck,
-      storagesStatistics: StoragesStatistics,
       diskCopy: DiskStorageCopyFiles,
       remoteDiskCopy: RemoteDiskStorageCopyFiles
   )(implicit uuidF: UUIDF): BatchCopy = new BatchCopy {
@@ -68,21 +66,9 @@ object BatchCopy {
 
     private def validateFilesForStorage(destStorage: Storage, sourcesBytes: NonEmptyList[Long]): IO[Unit] = {
       val maxSize = destStorage.storageValue.maxFileSize
-      for {
-        _ <- IO.raiseWhen(sourcesBytes.exists(_ > maxSize))(SourceFileTooLarge(maxSize, destStorage.id))
-        _ <- validateRemainingStorageCapacity(destStorage, sourcesBytes)
-      } yield ()
+      IO.raiseWhen(sourcesBytes.exists(_ > maxSize))(SourceFileTooLarge(maxSize, destStorage.id))
     }
 
-    private def validateRemainingStorageCapacity(destStorage: Storage, sourcesBytes: NonEmptyList[Long]) =
-      for {
-        spaceLeft <- storagesStatistics.getStorageAvailableSpace(destStorage)
-        totalSize  = sourcesBytes.toList.sum
-        _         <- spaceLeft
-                       .collectFirst { case s if s < totalSize => notEnoughSpace(totalSize, s, destStorage.id) }
-                       .getOrElse(IO.unit)
-      } yield ()
-
     private def fetchDiskCopyDetails(destStorage: DiskStorage, fileId: FileId)(implicit
         c: Caller
     ): IO[DiskCopyDetails] =
@@ -122,9 +108,6 @@ object BatchCopy {
 
     private def unsupported(tpe: StorageType) = IO.raiseError(CopyFileRejection.UnsupportedOperation(tpe))
 
-    private def notEnoughSpace(totalSize: Long, spaceLeft: Long, destStorage: Iri) =
-      IO.raiseError(TotalCopySizeTooLarge(totalSize, spaceLeft, destStorage))
-
     private def fetchFileAndValidateStorage(id: FileId)(implicit c: Caller) = {
       for {
         file          <- fetchFile.fetch(id)

diff --git a/...rc/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesConfig.scala b/...rc/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesConfig.scala
@@ -134,8 +134,6 @@ object StoragesConfig {
     *   the default permission required in order to upload a file to a disk storage
     * @param showLocation
     *   flag to decide whether or not to show the absolute location of the files in the metadata response
-    * @param defaultCapacity
-    *   the default capacity available to store the files
     * @param defaultMaxFileSize
     *   the default maximum allowed file size (in bytes) for uploaded files
     */
@@ -146,7 +144,6 @@ object StoragesConfig {
       defaultReadPermission: Permission,
       defaultWritePermission: Permission,
       showLocation: Boolean,
-      defaultCapacity: Option[Long],
       defaultMaxFileSize: Long
   ) extends StorageTypeEntryConfig
 

diff --git a/...ain/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesStatistics.scala b/...ain/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesStatistics.scala
@@ -4,7 +4,7 @@ import akka.http.scaladsl.model.Uri.Query
 import cats.effect.IO
 import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.EventMetricsProjection.eventMetricsIndex
 import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.ElasticSearchClient
-import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageStatEntry}
+import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageStatEntry
 import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri
 import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegment
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef
@@ -18,17 +18,6 @@ trait StoragesStatistics {
     */
   def get(idSegment: IdSegment, project: ProjectRef): IO[StorageStatEntry]
 
-  /**
-    * Retrieve remaining space on a storage if it has a capacity.
-    */
-  final def getStorageAvailableSpace(storage: Storage): IO[Option[Long]] =
-    storage.storageValue.capacity.fold(IO.none[Long]) { capacity =>
-      get(storage.id, storage.project)
-        .redeem(
-          _ => Some(capacity),
-          stat => Some(capacity - stat.spaceUsed)
-        )
-    }
 }
 
 object StoragesStatistics {

diff --git a/...in/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageFields.scala b/...in/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageFields.scala
@@ -87,8 +87,6 @@ object StorageFields {
     *   the permission required in order to download a file from this storage
     * @param writePermission
     *   the permission required in order to upload a file to this storage
-    * @param capacity
-    *   the capacity available (in bytes) to store files
     * @param maxFileSize
     *   the maximum allowed file size (in bytes) for uploaded files
     */
@@ -99,7 +97,6 @@ object StorageFields {
       volume: Option[AbsolutePath],
       readPermission: Option[Permission],
       writePermission: Option[Permission],
-      capacity: Option[Long],
       maxFileSize: Option[Long]
   ) extends StorageFields {
     override val tpe: StorageType = StorageType.DiskStorage
@@ -116,7 +113,6 @@ object StorageFields {
           volume.getOrElse(config.disk.defaultVolume),
           readPermission.getOrElse(config.disk.defaultReadPermission),
           writePermission.getOrElse(config.disk.defaultWritePermission),
-          capacity.orElse(config.disk.defaultCapacity),
           computeMaxFileSize(maxFileSize, config.disk.defaultMaxFileSize)
         )
       )

diff --git a/...ain/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageValue.scala b/...ain/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageValue.scala
@@ -48,12 +48,6 @@ sealed trait StorageValue extends Product with Serializable {
     */
   def algorithm: DigestAlgorithm
 
-  /**
-    * @return
-    *   the maximum allocated capacity for the storage
-    */
-  def capacity: Option[Long]
-
   /**
     * @return
     *   the maximum allowed file size (in bytes) for uploaded files
@@ -89,7 +83,6 @@ object StorageValue {
       volume: AbsolutePath,
       readPermission: Permission,
       writePermission: Permission,
-      capacity: Option[Long],
       maxFileSize: Long
   ) extends StorageValue {
 
@@ -111,10 +104,9 @@ object StorageValue {
         volume: AbsolutePath,
         readPermission: Permission,
         writePermission: Permission,
-        capacity: Option[Long],
         maxFileSize: Long
     ): DiskStorageValue =
-      DiskStorageValue(None, None, default, algorithm, volume, readPermission, writePermission, capacity, maxFileSize)
+      DiskStorageValue(None, None, default, algorithm, volume, readPermission, writePermission, maxFileSize)
 
   }
 
@@ -135,8 +127,8 @@ object StorageValue {
       maxFileSize: Long
   ) extends StorageValue {
 
-    override val tpe: StorageType       = StorageType.S3Storage
-    override val capacity: Option[Long] = None
+    override val tpe: StorageType = StorageType.S3Storage
+
   }
 
   object S3StorageValue {
@@ -182,8 +174,8 @@ object StorageValue {
       maxFileSize: Long
   ) extends StorageValue {
 
-    override val tpe: StorageType       = StorageType.RemoteDiskStorage
-    override val capacity: Option[Long] = None
+    override val tpe: StorageType = StorageType.RemoteDiskStorage
+
   }
 
   object RemoteDiskStorageValue {

diff --git a/delta/plugins/storage/src/test/resources/storages/database/disk-storage-created.json b/delta/plugins/storage/src/test/resources/storages/database/disk-storage-created.json
@@ -9,7 +9,6 @@
     "volume": "/tmp",
     "readPermission": "disk/read",
     "writePermission": "disk/write",
-    "capacity" : 1000,
     "maxFileSize": 50,
     "@type": "DiskStorageValue"
   },
@@ -21,7 +20,6 @@
     "default": true,
     "readPermission": "disk/read",
     "writePermission": "disk/write",
-    "capacity" : 1000,
     "maxFileSize": 50
   },
   "rev": 1,

diff --git a/delta/plugins/storage/src/test/resources/storages/database/disk-storage-updated.json b/delta/plugins/storage/src/test/resources/storages/database/disk-storage-updated.json
@@ -9,7 +9,6 @@
     "volume" : "/tmp",
     "readPermission" : "disk/read",
     "writePermission" : "disk/write",
-    "capacity" : 2000,
     "maxFileSize" : 40,
     "@type" : "DiskStorageValue"
   },
@@ -21,7 +20,6 @@
     "default" : true,
     "readPermission" : "disk/read",
     "writePermission" : "disk/write",
-    "capacity" : 1000,
     "maxFileSize" : 50
   },
   "rev" : 2,

diff --git a/delta/plugins/storage/src/test/resources/storages/disk-storage-expanded.json b/delta/plugins/storage/src/test/resources/storages/disk-storage-expanded.json
@@ -25,11 +25,6 @@
         "@value": true
       }
     ],
-    "https://bluebrain.github.io/nexus/vocabulary/capacity": [
-      {
-        "@value": 1000
-      }
-    ],
     "https://bluebrain.github.io/nexus/vocabulary/maxFileSize": [
       {
         "@value": 50

diff --git a/delta/plugins/storage/src/test/resources/storages/disk-storage.json b/delta/plugins/storage/src/test/resources/storages/disk-storage.json
@@ -9,6 +9,5 @@
   "default": true,
   "readPermission": "disk/read",
   "writePermission": "disk/write",
-  "capacity": 1000,
   "maxFileSize": 50
 }
diff --git a/delta/plugins/storage/src/test/resources/storages/sse/disk-storage-created.json b/delta/plugins/storage/src/test/resources/storages/sse/disk-storage-created.json
@@ -13,7 +13,6 @@
     "@type": "DiskStorage",
     "name": "diskName",
     "description": "diskDescription",
-    "capacity": 1000,
     "default": true,
     "maxFileSize": 50,
     "readPermission": "disk/read",

diff --git a/delta/plugins/storage/src/test/resources/storages/sse/disk-storage-updated.json b/delta/plugins/storage/src/test/resources/storages/sse/disk-storage-updated.json
@@ -13,7 +13,6 @@
     "@type": "DiskStorage",
     "name": "diskName",
     "description": "diskDescription",
-    "capacity": 1000,
     "default": true,
     "maxFileSize": 50,
     "readPermission": "disk/read",

diff --git a/delta/plugins/storage/src/test/resources/storages/storage-disk-state.json b/delta/plugins/storage/src/test/resources/storages/storage-disk-state.json
@@ -15,7 +15,6 @@
     "volume": "/tmp",
     "readPermission": "disk/read",
     "writePermission": "disk/write",
-    "capacity": 1000,
     "maxFileSize": 50,
     "@type": "DiskStorageValue"
   },
@@ -27,7 +26,6 @@
     "default": true,
     "readPermission": "disk/read",
     "writePermission": "disk/write",
-    "capacity": 1000,
     "maxFileSize": 50
   },
   "createdAt": "1970-01-01T00:00:00Z",

diff --git a/.../scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StorageScopeInitializationSpec.scala b/.../scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StorageScopeInitializationSpec.scala
@@ -64,7 +64,6 @@ class StorageScopeInitializationSpec
         volume = config.disk.defaultVolume,
         readPermission = config.disk.defaultReadPermission,
         writePermission = config.disk.defaultWritePermission,
-        capacity = config.disk.defaultCapacity,
         maxFileSize = config.disk.defaultMaxFileSize
       )
       resource.rev shouldEqual 1L

diff --git a/...torage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala b/...torage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala
@@ -167,7 +167,7 @@ class FilesSpec(fixture: RemoteStorageClientFixtures)
     "creating a file" should {
 
       "create storages for files" in {
-        val payload = diskFieldsJson deepMerge json"""{"capacity": 320, "maxFileSize": 300, "volume": "$path"}"""
+        val payload = diskFieldsJson deepMerge json"""{"maxFileSize": 300, "volume": "$path"}"""
         storages.create(diskId, projectRef, payload).accepted
 
         val payload2 =