Remove StoragesStatistics from Files

delta/plugins/storage/src/main/resources/storage.conf

@@ -21,9 +21,7 @@ plugins.storage {
       default-write-permission = "files/write"
       # flag to decide whether or not to show the absolute location of the files in the metadata response
       show-location = false
-      # the default capacity for storages (in bytes), by default no limit is defined
-      default-capacity = null
-      # the default maximum allowed file size (in bytes) for uploaded files. 10 GB 
+      # the default maximum allowed file size (in bytes) for uploaded files. 10 GB
       default-max-file-size = 10737418240
     }
     # S3 compatible storage configuration

delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala

@@ -180,7 +180,6 @@ class StoragePluginModule(priority: Int) extends ModuleDef {
         aclCheck: AclCheck,
         fetchContext: FetchContext,
         storages: Storages,
-        storagesStatistics: StoragesStatistics,
         xas: Transactors,
         clock: Clock[IO],
         uuidF: UUIDF,
@@ -191,7 +190,6 @@ class StoragePluginModule(priority: Int) extends ModuleDef {
         fetchContext,
         aclCheck,
         storages,
-        storagesStatistics,
         xas,
         cfg.files,
         fileOps,
@@ -220,12 +218,11 @@ class StoragePluginModule(priority: Int) extends ModuleDef {
         files: Files,
         storages: Storages,
         aclCheck: AclCheck,
-        storagesStatistics: StoragesStatistics,
         diskCopy: DiskStorageCopyFiles,
         remoteDiskCopy: RemoteDiskStorageCopyFiles,
         uuidF: UUIDF
     ) =>
-      BatchCopy.mk(files, storages, aclCheck, storagesStatistics, diskCopy, remoteDiskCopy)(uuidF)
+      BatchCopy.mk(files, storages, aclCheck, diskCopy, remoteDiskCopy)(uuidF)
   }
 
   make[BatchFiles].from {

delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StorageScopeInitialization.scala

@@ -74,7 +74,6 @@ object StorageScopeInitialization {
       volume = None,
       readPermission = None,
       writePermission = None,
-      capacity = None,
       maxFileSize = None
     )
     new StorageScopeInitialization(storages, serviceAccount, defaultStorageId, defaultFields)

delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala

@@ -20,7 +20,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejec
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{DigestAlgorithm, Storage, StorageRejection, StorageType}
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.{FetchAttributeRejection, FetchFileRejection, SaveFileRejection}
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations._
-import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{FetchStorage, Storages, StoragesStatistics}
+import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{FetchStorage, Storages}
 import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.ContextValue
 import ch.epfl.bluebrain.nexus.delta.sdk.AkkaSource
@@ -52,7 +52,6 @@ final class Files(
     aclCheck: AclCheck,
     fetchContext: FetchContext,
     storages: FetchStorage,
-    storagesStatistics: StoragesStatistics,
     fileOperations: FileOperations
 )(implicit uuidF: UUIDF)
     extends FetchFileStorage
@@ -77,6 +76,8 @@ final class Files(
     *   the http FormData entity
     * @param tag
     *   the optional tag this file is being created with, attached to the current revision
+    * @param metadata
+    *   the optional custom metadata provided by the user
     */
   def create(
       storageId: Option[IdSegment],
@@ -102,12 +103,12 @@ final class Files(
     *   the file identifier to expand as the iri of the file
     * @param storageId
     *   the optional storage identifier to expand as the id of the storage. When None, the default storage is used
-    * @param projectRef
-    *   the project where the file will belong
     * @param entity
     *   the http FormData entity
     * @param tag
     *   the optional tag this file is being created with, attached to the current revision
+    * @param metadata
+    *   the optional custom metadata provided by the user
     */
   def create(
       id: FileId,
@@ -132,10 +133,6 @@ final class Files(
     *   the optional storage identifier to expand as the id of the storage. When None, the default storage is used
     * @param projectRef
     *   the project where the file will belong
-    * @param filename
-    *   the optional filename to use
-    * @param mediaType
-    *   the optional media type to use
     * @param path
     *   the path where the file is located inside the storage
     * @param tag
@@ -162,12 +159,6 @@ final class Files(
     *   the file identifier to expand as the iri of the file
     * @param storageId
     *   the optional storage identifier to expand as the id of the storage. When None, the default storage is used
-    * @param projectRef
-    *   the project where the file will belong
-    * @param filename
-    *   the optional filename to use
-    * @param mediaType
-    *   the optional media type to use
     * @param path
     *   the path where the file is located inside the storage
     * @param tag
@@ -193,8 +184,6 @@ final class Files(
     *   the file identifier to expand as the iri of the file
     * @param storageId
     *   the optional storage identifier to expand as the id of the storage. When None, the default storage is used
-    * @param projectRef
-    *   the project where the file will belong
     * @param rev
     *   the current revision of the file
     * @param entity
@@ -236,14 +225,8 @@ final class Files(
     *   the file identifier to expand as the iri of the file
     * @param storageId
     *   the optional storage identifier to expand as the id of the storage. When None, the default storage is used
-    * @param projectRef
-    *   the project where the file will belong
     * @param rev
     *   the current revision of the file
-    * @param filename
-    *   the optional filename to use
-    * @param mediaType
-    *   the optional media type to use
     * @param path
     *   the path where the file is located inside the storage
     */
@@ -283,8 +266,6 @@ final class Files(
     *
     * @param id
     *   the file identifier to expand as the iri of the storage
-    * @param projectRef
-    *   the project where the file belongs
     * @param tag
     *   the tag name
     * @param tagRev
@@ -309,8 +290,6 @@ final class Files(
     *
     * @param id
     *   the identifier that will be expanded to the Iri of the file
-    * @param projectRef
-    *   the project reference where the file belongs
     * @param tag
     *   the tag name
     * @param rev
@@ -332,8 +311,6 @@ final class Files(
     *
     * @param id
     *   the file identifier to expand as the iri of the file
-    * @param projectRef
-    *   the project where the file belongs
     * @param rev
     *   the current revision of the file
     */
@@ -352,8 +329,6 @@ final class Files(
     *
     * @param id
     *   the file identifier to expand as the iri of the file
-    * @param projectRef
-    *   the project where the file belongs
     * @param rev
     *   the current revision of the file
     */
@@ -372,8 +347,6 @@ final class Files(
     *
     * @param id
     *   the identifier that will be expanded to the Iri of the file with its optional rev/tag
-    * @param project
-    *   the project where the storage belongs
     */
   def fetchContent(id: FileId)(implicit caller: Caller): IO[FileResponse] = {
     for {
@@ -480,17 +453,11 @@ final class Files(
       fileMetadata: Option[FileCustomMetadata]
   ): IO[FileAttributes] =
     for {
-      info            <- extractFormData(iri, storage, entity)
+      info            <- formDataExtractor(iri, entity, storage.storageValue.maxFileSize)
       description      = FileDescription.from(info, fileMetadata)
       storageMetadata <- saveFile(iri, storage, description, info.contents)
     } yield FileAttributes.from(description, storageMetadata)
 
-  private def extractFormData(iri: Iri, storage: Storage, entity: HttpEntity): IO[UploadedFileInformation] =
-    for {
-      storageAvailableSpace <- storagesStatistics.getStorageAvailableSpace(storage)
-      fi                    <- formDataExtractor(iri, entity, storage.storageValue.maxFileSize, storageAvailableSpace)
-    } yield fi
-
   private def saveFile(
       iri: Iri,
       storage: Storage,
@@ -754,7 +721,6 @@ object Files {
       fetchContext: FetchContext,
       aclCheck: AclCheck,
       storages: FetchStorage,
-      storagesStatistics: StoragesStatistics,
       xas: Transactors,
       config: FilesConfig,
       fileOps: FileOperations,
@@ -770,7 +736,6 @@ object Files {
       aclCheck,
       fetchContext,
       storages,
-      storagesStatistics,
       fileOps
     )
   }

delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FormDataExtractor.scala

@@ -28,19 +28,16 @@ sealed trait FormDataExtractor {
     * @param id
     *   the file id
     * @param entity
-    *   the Miltipart/FormData payload
+    *   the Multipart/FormData payload
     * @param maxFileSize
     *   the file size limit to be uploaded, provided by the storage
-    * @param storageAvailableSpace
-    *   the remaining available space on the storage
     * @return
     *   the file metadata. plus the entity with the file content
     */
   def apply(
       id: Iri,
       entity: HttpEntity,
-      maxFileSize: Long,
-      storageAvailableSpace: Option[Long]
+      maxFileSize: Long
   ): IO[UploadedFileInformation]
 }
 
@@ -79,13 +76,11 @@ object FormDataExtractor {
       override def apply(
           id: Iri,
           entity: HttpEntity,
-          maxFileSize: Long,
-          storageAvailableSpace: Option[Long]
+          maxFileSize: Long
       ): IO[UploadedFileInformation] = {
-        val sizeLimit = Math.min(storageAvailableSpace.getOrElse(Long.MaxValue), maxFileSize)
         for {
-          formData <- unmarshall(entity, sizeLimit)
-          fileOpt  <- extractFile(formData, maxFileSize, storageAvailableSpace)
+          formData <- unmarshall(entity, maxFileSize)
+          fileOpt  <- extractFile(formData, maxFileSize)
           file     <- IO.fromOption(fileOpt)(InvalidMultipartFieldName(id))
         } yield file
       }
@@ -110,8 +105,7 @@ object FormDataExtractor {
 
       private def extractFile(
           formData: FormData,
-          maxFileSize: Long,
-          storageAvailableSpace: Option[Long]
+          maxFileSize: Long
       ): IO[Option[UploadedFileInformation]] = IO
         .fromFuture(
           IO(
@@ -124,7 +118,7 @@ object FormDataExtractor {
         )
         .adaptError {
           case _: EntityStreamSizeException =>
-            FileTooLarge(maxFileSize, storageAvailableSpace)
+            FileTooLarge(maxFileSize)
           case NotARejection(th)            =>
             WrappedAkkaRejection(MalformedRequestContentRejection(th.getMessage, th))
         }

delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala

@@ -7,15 +7,14 @@ import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.FetchFileResource
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model._
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource
+import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.FetchStorage
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.{DiskStorage, RemoteDiskStorage}
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageType}
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection._
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{DiskCopyDetails, DiskStorageCopyFiles}
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.RemoteDiskStorageCopyFiles
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.RemoteDiskCopyDetails
-import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{FetchStorage, StoragesStatistics}
-import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri
 import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck
 import ch.epfl.bluebrain.nexus.delta.sdk.error.ServiceError.AuthorizationFailed
 import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller
@@ -32,7 +31,6 @@ object BatchCopy {
       fetchFile: FetchFileResource,
       fetchStorage: FetchStorage,
       aclCheck: AclCheck,
-      storagesStatistics: StoragesStatistics,
       diskCopy: DiskStorageCopyFiles,
       remoteDiskCopy: RemoteDiskStorageCopyFiles
   )(implicit uuidF: UUIDF): BatchCopy = new BatchCopy {
@@ -68,21 +66,9 @@ object BatchCopy {
 
     private def validateFilesForStorage(destStorage: Storage, sourcesBytes: NonEmptyList[Long]): IO[Unit] = {
       val maxSize = destStorage.storageValue.maxFileSize
-      for {
-        _ <- IO.raiseWhen(sourcesBytes.exists(_ > maxSize))(SourceFileTooLarge(maxSize, destStorage.id))
-        _ <- validateRemainingStorageCapacity(destStorage, sourcesBytes)
-      } yield ()
+      IO.raiseWhen(sourcesBytes.exists(_ > maxSize))(SourceFileTooLarge(maxSize, destStorage.id))
     }
 
-    private def validateRemainingStorageCapacity(destStorage: Storage, sourcesBytes: NonEmptyList[Long]) =
-      for {
-        spaceLeft <- storagesStatistics.getStorageAvailableSpace(destStorage)
-        totalSize  = sourcesBytes.toList.sum
-        _         <- spaceLeft
-                       .collectFirst { case s if s < totalSize => notEnoughSpace(totalSize, s, destStorage.id) }
-                       .getOrElse(IO.unit)
-      } yield ()
-
     private def fetchDiskCopyDetails(destStorage: DiskStorage, fileId: FileId)(implicit
         c: Caller
     ): IO[DiskCopyDetails] =
@@ -122,9 +108,6 @@ object BatchCopy {
 
     private def unsupported(tpe: StorageType) = IO.raiseError(CopyFileRejection.UnsupportedOperation(tpe))
 
-    private def notEnoughSpace(totalSize: Long, spaceLeft: Long, destStorage: Iri) =
-      IO.raiseError(TotalCopySizeTooLarge(totalSize, spaceLeft, destStorage))
-
     private def fetchFileAndValidateStorage(id: FileId)(implicit c: Caller) = {
       for {
         file          <- fetchFile.fetch(id)

delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileRejection.scala

@@ -145,11 +145,8 @@ object FileRejection {
     * Rejection returned when attempting to create/update a file with a Multipart/Form-Data payload that does not
     * contain a ''file'' fieldName
     */
-  final case class FileTooLarge(maxFileSize: Long, storageAvailableSpace: Option[Long])
-      extends FileRejection(
-        s"File size exceeds the max file size for the storage ($maxFileSize bytes)${storageAvailableSpace
-          .fold("") { r => s" or its remaining available space ($r bytes)" }}."
-      )
+  final case class FileTooLarge(maxFileSize: Long)
+      extends FileRejection(s"File size exceeds the max file size for the storage ($maxFileSize bytes).")
 
   /**
     * Rejection returned when attempting to create/update a file and the unmarshaller fails
@@ -277,7 +274,7 @@ object FileRejection {
       case FileNotFound(_, _)                                              => (StatusCodes.NotFound, Seq.empty)
       case ResourceAlreadyExists(_, _)                                     => (StatusCodes.Conflict, Seq.empty)
       case IncorrectRev(_, _)                                              => (StatusCodes.Conflict, Seq.empty)
-      case FileTooLarge(_, _)                                              => (StatusCodes.PayloadTooLarge, Seq.empty)
+      case FileTooLarge(_)                                                 => (StatusCodes.PayloadTooLarge, Seq.empty)
       case WrappedAkkaRejection(rej)                                       => (rej.status, rej.headers)
       case WrappedStorageRejection(rej)                                    => (rej.status, rej.headers)
       // If this happens it signifies a system problem rather than the user having made a mistake

delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesConfig.scala

@@ -134,8 +134,6 @@ object StoragesConfig {
     *   the default permission required in order to upload a file to a disk storage
     * @param showLocation
     *   flag to decide whether or not to show the absolute location of the files in the metadata response
-    * @param defaultCapacity
-    *   the default capacity available to store the files
     * @param defaultMaxFileSize
     *   the default maximum allowed file size (in bytes) for uploaded files
     */
@@ -146,7 +144,6 @@ object StoragesConfig {
       defaultReadPermission: Permission,
       defaultWritePermission: Permission,
       showLocation: Boolean,
-      defaultCapacity: Option[Long],
       defaultMaxFileSize: Long
   ) extends StorageTypeEntryConfig