Remove remote storage implementation (#5258)

* Remove remote storage implementation --------- Co-authored-by: Simon Dumas <[email protected]>
BlueBrain · Dec 3, 2024 · f2c3903 · f2c3903
1 parent b492d30
commit f2c3903
Show file tree

Hide file tree

Showing 87 changed files with 297 additions and 3,340 deletions.
diff --git a/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/VersionRoutesSpec.scala b/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/VersionRoutesSpec.scala
@@ -31,7 +31,7 @@ class VersionRoutesSpec extends BaseRouteSpec {
   }
 
   private val dependency2 = new ServiceDependency {
-    override def serviceDescription: IO[ServiceDescription] = IO.pure(ServiceDescription("remoteStorage", "1.0.0"))
+    override def serviceDescription: IO[ServiceDescription] = IO.pure(ServiceDescription("blazegraph", "1.0.0"))
   }
 
   private val descriptionConfig = DescriptionConfig(Name.unsafe("delta"), Name.unsafe("dev"))
@@ -82,7 +82,7 @@ class VersionRoutesSpec extends BaseRouteSpec {
             "delta": "${descriptionConfig.version}",
             "dependencies": {
               "elasticsearch": "unknown",
-              "remoteStorage": "1.0.0"
+              "blazegraph": "1.0.0"
             },
             "plugins": {
               "pluginA": "1.0",

diff --git a/.../scala/ch/epfl/bluebrain/nexus/delta/plugins/blazegraph/BlazegraphServiceDependency.scala b/.../scala/ch/epfl/bluebrain/nexus/delta/plugins/blazegraph/BlazegraphServiceDependency.scala
@@ -6,11 +6,10 @@ import ch.epfl.bluebrain.nexus.delta.kernel.dependency.ServiceDependency
 import ch.epfl.bluebrain.nexus.delta.plugins.blazegraph.client.BlazegraphClient
 
 /**
-  * Describes the remote storage [[ServiceDependency]] providing a way to extract the [[ServiceDescription]] from a
-  * remote storage calling its ''/version'' endpoint
+  * Describes the Blazegraph [[ServiceDependency]] providing a way to extract the [[ServiceDescription]] from its
+  * ''/status'' endpoint
   */
 class BlazegraphServiceDependency(client: BlazegraphClient) extends ServiceDependency {
 
-  override def serviceDescription: IO[ServiceDescription] =
-    client.serviceDescription
+  override def serviceDescription: IO[ServiceDescription] = client.serviceDescription
 }
diff --git a/.../ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/ElasticSearchServiceDependency.scala b/.../ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/ElasticSearchServiceDependency.scala
@@ -6,8 +6,8 @@ import ch.epfl.bluebrain.nexus.delta.kernel.dependency.ServiceDependency
 import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.ElasticSearchClient
 
 /**
-  * Describes the remote storage [[ServiceDependency]] providing a way to extract the [[ServiceDescription]] from a
-  * remote storage calling its ''/version'' endpoint
+  * Describes the Elasticsearch [[ServiceDependency]] providing a way to extract the [[ServiceDescription]] from
+  * Elasticsearch
   */
 class ElasticSearchServiceDependency(client: ElasticSearchClient) extends ServiceDependency {
 

diff --git a/delta/plugins/storage/src/main/resources/contexts/files.json b/delta/plugins/storage/src/main/resources/contexts/files.json
@@ -7,7 +7,6 @@
     "@vocab": "https://bluebrain.github.io/nexus/keywords/",
     "DiskStorage": "https://bluebrain.github.io/nexus/vocabulary/DiskStorage",
     "S3Storage": "https://bluebrain.github.io/nexus/vocabulary/S3Storage",
-    "RemoteDiskStorage": "https://bluebrain.github.io/nexus/vocabulary/RemoteDiskStorage",
     "File": "https://bluebrain.github.io/nexus/vocabulary/File",
     "_digest": "https://bluebrain.github.io/nexus/vocabulary/digest",
     "_bytes": "https://bluebrain.github.io/nexus/vocabulary/bytes",

diff --git a/delta/plugins/storage/src/main/resources/contexts/storages-metadata.json b/delta/plugins/storage/src/main/resources/contexts/storages-metadata.json
@@ -3,7 +3,6 @@
     "Storage": "https://bluebrain.github.io/nexus/vocabulary/Storage",
     "DiskStorage": "https://bluebrain.github.io/nexus/vocabulary/DiskStorage",
     "S3Storage": "https://bluebrain.github.io/nexus/vocabulary/S3Storage",
-    "RemoteDiskStorage": "https://bluebrain.github.io/nexus/vocabulary/RemoteDiskStorage",
     "_algorithm": "https://bluebrain.github.io/nexus/vocabulary/algorithm"
   },
   "@id": "https://bluebrain.github.io/nexus/contexts/storages-metadata.json"

diff --git a/delta/plugins/storage/src/main/resources/storage.conf b/delta/plugins/storage/src/main/resources/storage.conf
@@ -47,29 +47,6 @@ plugins.storage {
       default-bucket = "default-bucket-override-me"
       default-bucket = ${?DEFAULT_BUCKET}
     }
-    # Remote disk storage configuration
-    remote-disk {
-      # to enable remote storage
-      enabled = false
-      # the default endpoint
-      default-endpoint = "http://localhost:8084/v1"
-      # the default credentials for the endpoint
-      credentials {
-        type: "anonymous"
-      }
-      # the default digest algorithm
-      digest-algorithm = "SHA-256"
-      # the default permission required in order to download a file from a remote disk storage
-      default-read-permission = "resources/read"
-      # the default permission required in order to upload a file to a remote disk storage
-      default-write-permission = "files/write"
-      # flag to decide whether or not to show the absolute location of the files in the metadata response
-      show-location = true
-      # the default maximum allowed file size (in bytes) for uploaded files. 10 GB
-      default-max-file-size = 10737418240
-      # Retry delay for digest computation
-      digest-computation-retry-delay = 5s
-    }
     # the storages event log configuration
     event-log = ${app.defaults.event-log}
     # the storages pagination config

diff --git a/.../scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/RemoteStorageServiceDependency.scala b/.../scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/RemoteStorageServiceDependency.scala
diff --git a/...ge/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginConfig.scala b/...ge/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginConfig.scala
@@ -34,10 +34,7 @@ object StoragePluginConfig {
       .flatTap { config =>
         IO.whenA(config.storages.storageTypeConfig.amazon.isDefined) {
           logger.info("Amazon S3 storage is enabled")
-        } >>
-          IO.whenA(config.storages.storageTypeConfig.remoteDisk.isDefined) {
-            logger.info("Remote-disk storage is enabled")
-          }
+        }
       }
 
   implicit final val storagePluginConfig: ConfigReader[StoragePluginConfig] =

diff --git a/...ge/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala b/...ge/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala
@@ -4,7 +4,6 @@ import akka.actor.ActorSystem
 import akka.http.scaladsl.model.Uri.Path
 import akka.http.scaladsl.server.Directives.concat
 import cats.effect.{Clock, IO}
-import ch.epfl.bluebrain.nexus.delta.kernel.dependency.ServiceDependency
 import ch.epfl.bluebrain.nexus.delta.kernel.utils.{ClasspathResourceLoader, UUIDF}
 import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.ElasticSearchClient
 import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.config.ElasticSearchViewsConfig
@@ -13,31 +12,27 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.contexts.{files => fi
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model._
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.{DelegateFilesRoutes, FilesRoutes, LinkFilesRoutes}
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.schemas.{files => filesSchemaId}
-import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FileAttributesUpdateStream, Files, FormDataExtractor, MediaTypeDetector}
+import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{Files, FormDataExtractor, MediaTypeDetector}
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.{ShowFileLocation, StorageTypeConfig}
+import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages._
+import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.access.{S3StorageAccess, StorageAccess}
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.contexts.{storages => storageCtxId, storagesMetadata => storageMetaCtxId}
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model._
-import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.{FileOperations, LinkFileAction}
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.DiskFileOperations
-import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.RemoteDiskFileOperations
-import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.client.S3StorageClient
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.{S3FileOperations, S3LocationGenerator}
+import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.{FileOperations, LinkFileAction}
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.routes.StoragesRoutes
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.schemas.{storage => storagesSchemaId}
-import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages._
-import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.access.{RemoteStorageAccess, S3StorageAccess, StorageAccess}
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution}
 import ch.epfl.bluebrain.nexus.delta.rdf.utils.JsonKeyOrdering
 import ch.epfl.bluebrain.nexus.delta.sdk.IndexingAction.AggregateIndexingAction
 import ch.epfl.bluebrain.nexus.delta.sdk._
 import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck
-import ch.epfl.bluebrain.nexus.delta.sdk.auth.AuthTokenProvider
 import ch.epfl.bluebrain.nexus.delta.sdk.deletion.ProjectDeletionTask
 import ch.epfl.bluebrain.nexus.delta.sdk.directives.DeltaSchemeDirectives
 import ch.epfl.bluebrain.nexus.delta.sdk.fusion.FusionConfig
-import ch.epfl.bluebrain.nexus.delta.kernel.http.HttpClient
 import ch.epfl.bluebrain.nexus.delta.sdk.identities.Identities
 import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.ServiceAccount
 import ch.epfl.bluebrain.nexus.delta.sdk.jws.JWSPayloadHelper
@@ -49,7 +44,6 @@ import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ApiMappings
 import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution
 import ch.epfl.bluebrain.nexus.delta.sdk.sse.SseEncoder
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label
-import ch.epfl.bluebrain.nexus.delta.sourcing.stream.Supervisor
 import ch.epfl.bluebrain.nexus.delta.sourcing.{ScopedEventLog, Transactors}
 import com.typesafe.config.Config
 import izumi.distage.model.definition.{Id, ModuleDef}
@@ -67,10 +61,6 @@ class StoragePluginModule(priority: Int) extends ModuleDef {
 
   make[ShowFileLocation].from { cfg: StorageTypeConfig => cfg.showFileLocation }
 
-  make[HttpClient].named("storage").from { (as: ActorSystem) =>
-    HttpClient.noRetry(compression = false)(as)
-  }
-
   make[S3StorageClient].fromResource { (cfg: StoragePluginConfig) =>
     S3StorageClient.resource(cfg.storages.storageTypeConfig.amazon)
   }
@@ -80,9 +70,7 @@ class StoragePluginModule(priority: Int) extends ModuleDef {
     new S3LocationGenerator(prefix)
   }
 
-  make[StorageAccess].from { (remoteClient: RemoteDiskStorageClient, s3Client: S3StorageClient) =>
-    StorageAccess(RemoteStorageAccess(remoteClient), S3StorageAccess(s3Client))
-  }
+  make[StorageAccess].from { (s3Client: S3StorageClient) => StorageAccess(S3StorageAccess(s3Client)) }
 
   make[Storages]
     .fromEffect {
@@ -179,17 +167,13 @@ class StoragePluginModule(priority: Int) extends ModuleDef {
     DiskFileOperations.mk(as, uuidF)
   }
 
-  make[RemoteDiskFileOperations].from { (client: RemoteDiskStorageClient, uuidF: UUIDF) =>
-    RemoteDiskFileOperations.mk(client)(uuidF)
-  }
-
   make[S3FileOperations].from {
     (client: S3StorageClient, locationGenerator: S3LocationGenerator, uuidF: UUIDF, as: ActorSystem) =>
       S3FileOperations.mk(client, locationGenerator)(as, uuidF)
   }
 
-  make[FileOperations].from { (disk: DiskFileOperations, remoteDisk: RemoteDiskFileOperations, s3: S3FileOperations) =>
-    FileOperations.apply(disk, remoteDisk, s3)
+  make[FileOperations].from { (disk: DiskFileOperations, s3: S3FileOperations) =>
+    FileOperations.apply(disk, s3)
   }
 
   make[MediaTypeDetector].from { (cfg: StoragePluginConfig) =>
@@ -226,11 +210,6 @@ class StoragePluginModule(priority: Int) extends ModuleDef {
       )(uuidF)
   }
 
-  make[FileAttributesUpdateStream].fromEffect {
-    (files: Files, storages: Storages, storageTypeConfig: StorageTypeConfig, supervisor: Supervisor) =>
-      FileAttributesUpdateStream.start(files, storages, storageTypeConfig.remoteDisk, supervisor)
-  }
-
   make[FilesRoutes].from {
     (
         showLocation: ShowFileLocation,
@@ -302,26 +281,6 @@ class StoragePluginModule(priority: Int) extends ModuleDef {
 
   many[ResourceShift[_, _, _]].ref[File.Shift]
 
-  make[RemoteDiskStorageClient].from {
-    (
-        client: HttpClient @Id("storage"),
-        as: ActorSystem,
-        authTokenProvider: AuthTokenProvider,
-        cfg: StorageTypeConfig
-    ) =>
-      RemoteDiskStorageClient(client, authTokenProvider, cfg.remoteDisk)(as)
-  }
-
-  many[ServiceDependency].addSet {
-    (
-        cfg: StorageTypeConfig,
-        remoteStorageClient: RemoteDiskStorageClient
-    ) =>
-      cfg.remoteDisk.fold(Set.empty[ServiceDependency]) { _ =>
-        Set(new RemoteStorageServiceDependency(remoteStorageClient))
-      }
-  }
-
   many[ScopeInitialization].addSet { (storages: Storages, serviceAccount: ServiceAccount, cfg: StoragePluginConfig) =>
     Option.when(cfg.enableDefaultCreation)(StorageScopeInitialization(storages, serviceAccount, cfg.defaults)).toSet
   }

diff --git a/...cala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileAttributesUpdateStream.scala b/...cala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileAttributesUpdateStream.scala