diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala index 8759b26549..4ebc90d8b1 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala @@ -1,7 +1,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage import akka.actor.typed.ActorSystem -import akka.http.scaladsl.model.Uri +import akka.http.scaladsl.model.Uri.Path import cats.effect.{Clock, IO} import ch.epfl.bluebrain.nexus.delta.kernel.utils.{ClasspathResourceLoader, TransactionalFileCopier, UUIDF} import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.ElasticSearchClient @@ -74,7 +74,7 @@ class StoragePluginModule(priority: Int) extends ModuleDef { } make[S3LocationGenerator].from { (cfg: StoragePluginConfig) => - val prefix: Uri = cfg.storages.storageTypeConfig.amazon.flatMap(_.prefix).getOrElse(Uri.Empty) + val prefix: Path = cfg.storages.storageTypeConfig.amazon.flatMap(_.prefix).getOrElse(Path.Empty) new S3LocationGenerator(prefix) } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesConfig.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesConfig.scala index 41b47a7eab..5e9ec45aee 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesConfig.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesConfig.scala @@ -1,6 +1,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages import akka.http.scaladsl.model.Uri +import akka.http.scaladsl.model.Uri.Path import cats.implicits._ import ch.epfl.bluebrain.nexus.delta.kernel.Secret import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.StorageTypeConfig @@ -181,9 +182,9 @@ object StoragesConfig { showLocation: Boolean, defaultMaxFileSize: Long, defaultBucket: String, - prefix: Option[Uri] + prefix: Option[Path] ) extends StorageTypeEntryConfig { - val prefixUri: Uri = prefix.getOrElse(Uri.Empty) + val prefixPath: Path = prefix.getOrElse(Path.Empty) } /** @@ -218,13 +219,16 @@ object StoragesConfig { implicit private val uriConverter: ConfigConvert[Uri] = ConfigConvert.viaString[Uri](catchReadError(Uri(_)), _.toString) + implicit private val pathConverter: ConfigConvert[Path] = + ConfigConvert.viaString[Path](catchReadError(Path(_)), _.toString) + implicit private val permissionConverter: ConfigConvert[Permission] = ConfigConvert.viaString[Permission](optF(Permission(_).toOption), _.toString) implicit private val digestAlgConverter: ConfigConvert[DigestAlgorithm] = ConfigConvert.viaString[DigestAlgorithm](optF(DigestAlgorithm(_)), _.toString) - implicit private val pathConverter: ConfigConvert[AbsolutePath] = + implicit private val absolutePathConverter: ConfigConvert[AbsolutePath] = ConfigConvert.viaString[AbsolutePath]( str => AbsolutePath(str).leftMap(err => CannotConvert(str, "AbsolutePath", err)), _.toString diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3LocationGenerator.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3LocationGenerator.scala index 34143294e1..941f345056 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3LocationGenerator.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3LocationGenerator.scala @@ -1,17 +1,20 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3 import akka.http.scaladsl.model.Uri +import akka.http.scaladsl.model.Uri.Path import ch.epfl.bluebrain.nexus.delta.rdf.syntax.uriSyntax import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef import java.util.UUID -class S3LocationGenerator(prefix: Uri) { +class S3LocationGenerator(prefix: Path) { private def uuidToPath(uuid: UUID) = Uri.Path(uuid.toString.toLowerCase.takeWhile(_ != '-').mkString("/")) def file(project: ProjectRef, uuid: UUID, filename: String): Uri = - prefix / project.organization.toString / project.project.toString / "files" / uuidToPath(uuid) / filename + Uri.Empty / prefix / project.organization.toString / project.project.toString / "files" / uuidToPath( + uuid + ) / filename } diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/FileOperationsMock.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/FileOperationsMock.scala index 31eaeeb76a..be1adaa34f 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/FileOperationsMock.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/FileOperationsMock.scala @@ -2,6 +2,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks import akka.actor.ActorSystem import akka.http.scaladsl.model.Uri +import akka.http.scaladsl.model.Uri.Path import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.UploadedFileInformation @@ -30,7 +31,7 @@ object FileOperationsMock { FileOperations.mk( DiskFileOperations.mk, RemoteDiskFileOperations.mk(RemoteDiskStorageClientDisabled), - S3FileOperations.mk(S3StorageClient.disabled, new S3LocationGenerator(Uri.Empty)) + S3FileOperations.mk(S3StorageClient.disabled, new S3LocationGenerator(Path.Empty)) ) def unimplemented: FileOperations = new FileOperations { diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/LocalStackS3StorageClient.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/LocalStackS3StorageClient.scala index 7795ce9b65..d7d866cd03 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/LocalStackS3StorageClient.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/LocalStackS3StorageClient.scala @@ -53,7 +53,7 @@ object LocalStackS3StorageClient { showLocation = false, defaultMaxFileSize = 1, defaultBucket = defaultBucket, - prefix = Some(prefix) + prefix = Some(Uri.Path(prefix)) ) (S3StorageClient.unsafe(client), client, conf) } diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3FileOperationsSuite.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3FileOperationsSuite.scala index 287fd65002..46c952f523 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3FileOperationsSuite.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3FileOperationsSuite.scala @@ -43,7 +43,7 @@ class S3FileOperationsSuite localStackS3Client() implicit private lazy val as: ActorSystem = actorSystem() - private lazy val locationGenerator = new S3LocationGenerator(conf.prefixUri) + private lazy val locationGenerator = new S3LocationGenerator(conf.prefixPath) private lazy val fileOps = S3FileOperations.mk(s3StorageClient, locationGenerator) private def makeDigest(content: String): ComputedDigest = { @@ -54,7 +54,8 @@ class S3FileOperationsSuite private def expectedPath(proj: ProjectRef, filename: String): Uri.Path = Uri.Path(s"$proj/files/${randomUuid.toString.takeWhile(_ != '-').mkString("/")}/$filename") - private def expectedLocation(proj: ProjectRef, filename: String): Uri = conf.prefixUri / expectedPath(proj, filename) + private def expectedLocation(proj: ProjectRef, filename: String): Uri = + Uri.Empty / conf.prefixPath / expectedPath(proj, filename) test("List objects in an existing bucket") { givenAnS3Bucket { bucket => diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3LocationGeneratorSuite.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3LocationGeneratorSuite.scala index 4ba507fb2a..3168f1d027 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3LocationGeneratorSuite.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3LocationGeneratorSuite.scala @@ -1,6 +1,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3 import akka.http.scaladsl.model.Uri +import akka.http.scaladsl.model.Uri.Path import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef import ch.epfl.bluebrain.nexus.testkit.mu.NexusSuite @@ -9,7 +10,8 @@ import java.util.UUID class S3LocationGeneratorSuite extends NexusSuite { test("Generate the expected uri") { - val prefix = Uri("/prefix") + val prefix = Path("/prefix") + println(prefix) val project = ProjectRef.unsafe("org", "project") val uuid = UUID.fromString("12345678-b2e3-40b9-93de-c809415d7640") val filename = "cat.gif" diff --git a/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/instances/UriInstances.scala b/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/instances/UriInstances.scala index 2c7afc14a8..8a91218160 100644 --- a/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/instances/UriInstances.scala +++ b/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/instances/UriInstances.scala @@ -30,6 +30,11 @@ trait UriInstances { .toEither .leftMap(err => CannotConvert(str, classOf[Uri].getSimpleName, err.getMessage)) ) + + implicit val pathConfigReader: ConfigReader[Uri.Path] = ConfigReader.fromString(str => + Try(Uri.Path(str)).toEither + .leftMap(err => CannotConvert(str, classOf[Uri.Path].getSimpleName, err.getMessage)) + ) } object UriInstances extends UriInstances diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/FileProcessingConfig.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/FileProcessingConfig.scala index 76e74c4be9..6e9ffb6545 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/FileProcessingConfig.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/config/FileProcessingConfig.scala @@ -1,6 +1,6 @@ package ch.epfl.bluebrain.nexus.ship.config -import akka.http.scaladsl.model.Uri +import akka.http.scaladsl.model.Uri.Path import pureconfig.ConfigReader import ch.epfl.bluebrain.nexus.delta.sdk.instances._ import pureconfig.generic.semiauto.deriveReader @@ -8,7 +8,7 @@ import pureconfig.generic.semiauto.deriveReader final case class FileProcessingConfig( importBucket: String, targetBucket: String, - prefix: Option[Uri], + prefix: Option[Path], skipFileEvents: Boolean ) diff --git a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/files/FileCopier.scala b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/files/FileCopier.scala index abdfc0fbe9..db6f66f82d 100644 --- a/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/files/FileCopier.scala +++ b/ship/src/main/scala/ch/epfl/bluebrain/nexus/ship/files/FileCopier.scala @@ -1,6 +1,7 @@ package ch.epfl.bluebrain.nexus.ship.files import akka.http.scaladsl.model.Uri +import akka.http.scaladsl.model.Uri.Path import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.Logger import ch.epfl.bluebrain.nexus.delta.kernel.utils.UrlUtils @@ -42,7 +43,7 @@ object FileCopier { ): FileCopier = { val importBucket = config.importBucket val targetBucket = config.targetBucket - val locationGenerator = new S3LocationGenerator(config.prefix.getOrElse(Uri.Empty)) + val locationGenerator = new S3LocationGenerator(config.prefix.getOrElse(Path.Empty)) (project: ProjectRef, attributes: FileAttributes) => { val origin = attributes.path val patchedFileName = if (attributes.filename.isEmpty) "file" else attributes.filename diff --git a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigFixtures.scala b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigFixtures.scala index dddfbe6bd6..949e894523 100644 --- a/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigFixtures.scala +++ b/ship/src/test/scala/ch/epfl/bluebrain/nexus/ship/config/ShipConfigFixtures.scala @@ -1,10 +1,11 @@ package ch.epfl.bluebrain.nexus.ship.config +import akka.http.scaladsl.model.Uri import ch.epfl.bluebrain.nexus.delta.kernel.Secret import ch.epfl.bluebrain.nexus.delta.plugins.storage.files import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.S3StorageConfig import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{permissions, StorageFixtures, StoragesConfig} -import ch.epfl.bluebrain.nexus.delta.rdf.syntax.{iriStringContextSyntax, uriStringContextSyntax} +import ch.epfl.bluebrain.nexus.delta.rdf.syntax.iriStringContextSyntax import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.ServiceAccount import ch.epfl.bluebrain.nexus.delta.sdk.model.{BaseUri, ServiceAccountConfig} import ch.epfl.bluebrain.nexus.delta.sdk.{ConfigFixtures, Defaults} @@ -59,7 +60,7 @@ trait ShipConfigFixtures extends ConfigFixtures with StorageFixtures with Classp FileProcessingConfig( importBucket, targetBucket, - Some(uri"/prefix"), + Some(Uri.Path("/prefix")), skipFileEvents = false ), IriPatcherConfig(enabled = false, iri"https://bbp.epfl.ch/", iri"https:/openbrainplatform.com/"), diff --git a/tests/docker/config/delta-postgres.conf b/tests/docker/config/delta-postgres.conf index 31002aa6d7..7ebbfaeab8 100644 --- a/tests/docker/config/delta-postgres.conf +++ b/tests/docker/config/delta-postgres.conf @@ -156,7 +156,7 @@ plugins { default-access-key = "MY_ACCESS_KEY" default-secret-key = "CHUTCHUT" default-bucket = "mydefaultbucket" - prefix = "myprefix" + prefix = "/myprefix" } } diff --git a/tests/src/test/resources/application.conf b/tests/src/test/resources/application.conf index 828e899728..2990f1749f 100644 --- a/tests/src/test/resources/application.conf +++ b/tests/src/test/resources/application.conf @@ -21,6 +21,6 @@ storage { s-3 { access-key = "MY_ACCESS_KEY" secret-key = "CHUTCHUT" - prefix = "myprefix" + prefix = "/myprefix" } } \ No newline at end of file diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/S3StorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/S3StorageSpec.scala index 476e432f71..e12524ad4f 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/S3StorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/S3StorageSpec.scala @@ -231,7 +231,7 @@ class S3StorageSpec extends StorageSpec { private def assertThereIsAFileInS3WithAtLocation(location: String): Assertion = { s3Client - .listObjectsV2(ListObjectsV2Request.builder.bucket(bucket).prefix(s"myprefix/$projectRef/files").build) + .listObjectsV2(ListObjectsV2Request.builder.bucket(bucket).prefix(s"/myprefix/$projectRef/files").build) .map(_.contents.asScala.map(_.key())) .map(keys => keys should contain(location)) .accepted