From 7e8fe4e242c9dfd0d789d52415f9619422ceed60 Mon Sep 17 00:00:00 2001 From: Oliver <20188437+olivergrabinski@users.noreply.github.com> Date: Tue, 9 Apr 2024 16:38:13 +0200 Subject: [PATCH 1/2] Add logback.xml file (#4858) --- build.sbt | 2 +- ship/src/main/resources/logback.xml | 30 +++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 ship/src/main/resources/logback.xml diff --git a/build.sbt b/build.sbt index f2a86ca307..96dd883a3a 100755 --- a/build.sbt +++ b/build.sbt @@ -753,7 +753,7 @@ lazy val ship = project tests % "test->compile;test->test" ) .settings( - libraryDependencies ++= Seq(declineEffect), + libraryDependencies ++= Seq(declineEffect, logback), addCompilerPlugin(betterMonadicFor), run / fork := true, buildInfoKeys := Seq[BuildInfoKey](version), diff --git a/ship/src/main/resources/logback.xml b/ship/src/main/resources/logback.xml new file mode 100644 index 0000000000..d7f02b4a6a --- /dev/null +++ b/ship/src/main/resources/logback.xml @@ -0,0 +1,30 @@ + + + + + + + %d{yyyy-MM-dd HH:mm:ss} %-5level %logger{36} - %msg%n + + + + + + + + %d{yyyy-MM-dd HH:mm:ss} %-5level [%traceToken] %logger{36} - %msg%n + + + + + + + + + + + + + + + \ No newline at end of file From c4f01af0fe631e0dbdc4f9ac7f576d20326ead4d Mon Sep 17 00:00:00 2001 From: Daniel Bell Date: Wed, 10 Apr 2024 08:15:08 +0100 Subject: [PATCH 2/2] Remove endpoint and region from the S3 Storage instances (#4859) --- .../storage/storages/model/StorageFields.scala | 14 -------------- .../storage/storages/model/StorageValue.scala | 8 -------- .../storages/database/s3-storage-created.json | 4 ---- .../storages/database/s3-storage-updated.json | 4 ---- .../resources/storages/s3-storage-expanded.json | 10 ---------- .../resources/storages/s3-storage-fetched.json | 2 -- .../src/test/resources/storages/s3-storage.json | 2 -- .../resources/storages/sse/s3-storage-created.json | 2 -- .../resources/storages/sse/s3-storage-updated.json | 2 -- .../test/resources/storages/storage-s3-state.json | 4 ---- .../plugins/storage/storages/StorageFixtures.scala | 3 +-- .../storage/storages/model/StorageFieldsSpec.scala | 2 +- .../operations/s3/S3StorageFetchSaveSpec.scala | 2 -- .../main/paradox/docs/delta/api/storages-api.md | 4 ---- .../test/resources/kg/storages/s3-response.json | 1 - tests/src/test/resources/kg/storages/s3.json | 3 +-- .../nexus/tests/kg/files/S3StorageSpec.scala | 12 +++--------- 17 files changed, 6 insertions(+), 73 deletions(-) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageFields.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageFields.scala index d9640a1a08..41f907dbf9 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageFields.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageFields.scala @@ -1,11 +1,9 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model -import akka.http.scaladsl.model.Uri import ch.epfl.bluebrain.nexus.delta.kernel.Secret import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.StorageTypeConfig import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageValue.{DiskStorageValue, RemoteDiskStorageValue, S3StorageValue} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri -import ch.epfl.bluebrain.nexus.delta.sdk.implicits._ import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.decoder.configuration.semiauto.deriveConfigJsonLdDecoder import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.decoder.{Configuration => JsonLdConfiguration, JsonLdDecoder} @@ -131,14 +129,6 @@ object StorageFields { * ''true'' if this store is the project's default, ''false'' otherwise * @param bucket * the S3 compatible bucket - * @param endpoint - * the endpoint, either a domain or a full URL - * @param accessKey - * the AWS access key ID - * @param secretKey - * the AWS secret key - * @param region - * the AWS region * @param readPermission * the permission required in order to download a file from this storage * @param writePermission @@ -151,8 +141,6 @@ object StorageFields { description: Option[String], default: Boolean, bucket: String, - endpoint: Option[Uri], - region: Option[Region], readPermission: Option[Permission], writePermission: Option[Permission], maxFileSize: Option[Long] @@ -169,8 +157,6 @@ object StorageFields { default, cfg.digestAlgorithm, bucket, - endpoint.orElse(Some(cfg.defaultEndpoint)), - region, readPermission.getOrElse(cfg.defaultReadPermission), writePermission.getOrElse(cfg.defaultWritePermission), computeMaxFileSize(maxFileSize, cfg.defaultMaxFileSize) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageValue.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageValue.scala index 71b7b98dcd..35ba507013 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageValue.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageValue.scala @@ -1,9 +1,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model -import akka.http.scaladsl.model.Uri import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords -import ch.epfl.bluebrain.nexus.delta.sdk.implicits._ import ch.epfl.bluebrain.nexus.delta.sdk.permissions.model.Permission import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Label, ProjectRef} import io.circe.generic.extras.Configuration @@ -132,8 +130,6 @@ object StorageValue { default: Boolean, algorithm: DigestAlgorithm, bucket: String, - endpoint: Option[Uri], - region: Option[Region], readPermission: Permission, writePermission: Permission, maxFileSize: Long @@ -153,8 +149,6 @@ object StorageValue { default: Boolean, algorithm: DigestAlgorithm, bucket: String, - endpoint: Option[Uri], - region: Option[Region], readPermission: Permission, writePermission: Permission, maxFileSize: Long @@ -165,8 +159,6 @@ object StorageValue { default, algorithm, bucket, - endpoint, - region, readPermission, writePermission, maxFileSize diff --git a/delta/plugins/storage/src/test/resources/storages/database/s3-storage-created.json b/delta/plugins/storage/src/test/resources/storages/database/s3-storage-created.json index 403c80e44c..920947eb60 100644 --- a/delta/plugins/storage/src/test/resources/storages/database/s3-storage-created.json +++ b/delta/plugins/storage/src/test/resources/storages/database/s3-storage-created.json @@ -7,9 +7,7 @@ "description": "s3description", "algorithm" : "SHA-256", "bucket" : "mybucket", - "endpoint" : "http://localhost", "readPermission" : "s3/read", - "region": "eu-west-1", "writePermission" : "s3/write", "maxFileSize" : 51, "@type" : "S3StorageValue" @@ -20,9 +18,7 @@ "description": "s3description", "default" : true, "bucket" : "mybucket", - "endpoint" : "http://localhost", "readPermission" : "s3/read", - "region": "eu-west-1", "writePermission" : "s3/write", "maxFileSize" : 51 }, diff --git a/delta/plugins/storage/src/test/resources/storages/database/s3-storage-updated.json b/delta/plugins/storage/src/test/resources/storages/database/s3-storage-updated.json index 2ae1897aa6..bfe9036835 100644 --- a/delta/plugins/storage/src/test/resources/storages/database/s3-storage-updated.json +++ b/delta/plugins/storage/src/test/resources/storages/database/s3-storage-updated.json @@ -7,9 +7,7 @@ "description": "s3description", "algorithm" : "SHA-256", "bucket" : "mybucket2", - "endpoint" : "http://localhost", "readPermission" : "s3/read", - "region": "eu-west-1", "writePermission" : "s3/write", "maxFileSize" : 41, "@type" : "S3StorageValue" @@ -20,9 +18,7 @@ "description": "s3description", "default" : true, "bucket" : "mybucket", - "endpoint" : "http://localhost", "readPermission" : "s3/read", - "region": "eu-west-1", "writePermission" : "s3/write", "maxFileSize" : 51 }, diff --git a/delta/plugins/storage/src/test/resources/storages/s3-storage-expanded.json b/delta/plugins/storage/src/test/resources/storages/s3-storage-expanded.json index 91d4634fa6..c2b8cbfc10 100644 --- a/delta/plugins/storage/src/test/resources/storages/s3-storage-expanded.json +++ b/delta/plugins/storage/src/test/resources/storages/s3-storage-expanded.json @@ -30,11 +30,6 @@ "@value": true } ], - "https://bluebrain.github.io/nexus/vocabulary/endpoint": [ - { - "@value": "http://localhost" - } - ], "https://bluebrain.github.io/nexus/vocabulary/maxFileSize": [ { "@value": 51 @@ -45,11 +40,6 @@ "@value": "s3/read" } ], - "https://bluebrain.github.io/nexus/vocabulary/region": [ - { - "@value": "eu-west-1" - } - ], "https://bluebrain.github.io/nexus/vocabulary/writePermission": [ { "@value": "s3/write" diff --git a/delta/plugins/storage/src/test/resources/storages/s3-storage-fetched.json b/delta/plugins/storage/src/test/resources/storages/s3-storage-fetched.json index 8a86b8952b..4bddd08a14 100644 --- a/delta/plugins/storage/src/test/resources/storages/s3-storage-fetched.json +++ b/delta/plugins/storage/src/test/resources/storages/s3-storage-fetched.json @@ -24,9 +24,7 @@ "default" : true, "name": "s3name", "description": "s3description", - "endpoint" : "http://localhost", "maxFileSize" : 51, "readPermission" : "s3/read", - "region" : "eu-west-1", "writePermission" : "s3/write" } \ No newline at end of file diff --git a/delta/plugins/storage/src/test/resources/storages/s3-storage.json b/delta/plugins/storage/src/test/resources/storages/s3-storage.json index c873bfbb6c..020650b969 100644 --- a/delta/plugins/storage/src/test/resources/storages/s3-storage.json +++ b/delta/plugins/storage/src/test/resources/storages/s3-storage.json @@ -7,9 +7,7 @@ "name": "s3name", "description": "s3description", "bucket": "mybucket", - "endpoint": "http://localhost", "readPermission": "s3/read", - "region": "eu-west-1", "writePermission": "s3/write", "maxFileSize": 51 } \ No newline at end of file diff --git a/delta/plugins/storage/src/test/resources/storages/sse/s3-storage-created.json b/delta/plugins/storage/src/test/resources/storages/sse/s3-storage-created.json index 350cf75ab8..b9176c4aff 100644 --- a/delta/plugins/storage/src/test/resources/storages/sse/s3-storage-created.json +++ b/delta/plugins/storage/src/test/resources/storages/sse/s3-storage-created.json @@ -15,10 +15,8 @@ "description": "s3description", "bucket": "mybucket", "default": true, - "endpoint": "http://localhost", "maxFileSize": 51, "readPermission": "s3/read", - "region": "eu-west-1", "writePermission": "s3/write" }, "_storageId": "https://bluebrain.github.io/nexus/vocabulary/s3-storage", diff --git a/delta/plugins/storage/src/test/resources/storages/sse/s3-storage-updated.json b/delta/plugins/storage/src/test/resources/storages/sse/s3-storage-updated.json index 5d0ef9765f..cc35c47581 100644 --- a/delta/plugins/storage/src/test/resources/storages/sse/s3-storage-updated.json +++ b/delta/plugins/storage/src/test/resources/storages/sse/s3-storage-updated.json @@ -15,10 +15,8 @@ "description": "s3description", "bucket": "mybucket", "default": true, - "endpoint": "http://localhost", "maxFileSize": 51, "readPermission": "s3/read", - "region": "eu-west-1", "writePermission": "s3/write" }, "_storageId": "https://bluebrain.github.io/nexus/vocabulary/s3-storage", diff --git a/delta/plugins/storage/src/test/resources/storages/storage-s3-state.json b/delta/plugins/storage/src/test/resources/storages/storage-s3-state.json index 17b65e9ad1..6a1203ce85 100644 --- a/delta/plugins/storage/src/test/resources/storages/storage-s3-state.json +++ b/delta/plugins/storage/src/test/resources/storages/storage-s3-state.json @@ -13,8 +13,6 @@ "description": "s3description", "algorithm": "SHA-256", "bucket": "mybucket", - "endpoint": "http://localhost", - "region": "eu-west-1", "readPermission": "s3/read", "writePermission": "s3/write", "maxFileSize": 51, @@ -26,9 +24,7 @@ "description": "s3description", "default": true, "bucket": "mybucket", - "endpoint": "http://localhost", "readPermission": "s3/read", - "region": "eu-west-1", "writePermission": "s3/write", "maxFileSize": 51 }, diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StorageFixtures.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StorageFixtures.scala index 9a3a94e40c..ba4718230a 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StorageFixtures.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StorageFixtures.scala @@ -13,7 +13,6 @@ import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label import ch.epfl.bluebrain.nexus.testkit.CirceLiteral import ch.epfl.bluebrain.nexus.testkit.minio.MinioDocker import ch.epfl.bluebrain.nexus.testkit.scalatest.ClasspathResources -import software.amazon.awssdk.regions.Region import java.nio.file.{Files, Paths} import scala.concurrent.duration._ @@ -42,7 +41,7 @@ trait StorageFixtures extends CirceLiteral { val diskVal = diskFields.toValue(config).get val diskFieldsUpdate = DiskStorageFields(Some("diskName"), Some("diskDescription"), default = false, Some(tmpVolume), Some(Permission.unsafe("disk/read")), Some(Permission.unsafe("disk/write")), Some(2000), Some(40)) val diskValUpdate = diskFieldsUpdate.toValue(config).get - val s3Fields = S3StorageFields(Some("s3name"), Some("s3description"), default = true, "mybucket", Some("http://localhost"), Some(Region.EU_WEST_1), Some(Permission.unsafe("s3/read")), Some(Permission.unsafe("s3/write")), Some(51)) + val s3Fields = S3StorageFields(Some("s3name"), Some("s3description"), default = true, "mybucket", Some(Permission.unsafe("s3/read")), Some(Permission.unsafe("s3/write")), Some(51)) val s3Val = s3Fields.toValue(config).get val remoteFields = RemoteDiskStorageFields(Some("remoteName"), Some("remoteDescription"), default = true, Label.unsafe("myfolder"), Some(Permission.unsafe("remote/read")), Some(Permission.unsafe("remote/write")), Some(52)) val remoteVal = remoteFields.toValue(config).get diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageFieldsSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageFieldsSpec.scala index 21af6f81d3..d7fc6c8fa0 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageFieldsSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageFieldsSpec.scala @@ -63,7 +63,7 @@ class StorageFieldsSpec extends CatsEffectSpec with RemoteContextResolutionFixtu "region" ) sourceDecoder(pc, jsonNoDefaults).accepted._2 shouldEqual - S3StorageFields(None, None, default = true, "mybucket", None, None, None, None, None) + S3StorageFields(None, None, default = true, "mybucket", None, None, None) } } diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3StorageFetchSaveSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3StorageFetchSaveSpec.scala index 0ebf790eb8..ea37d16f69 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3StorageFetchSaveSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/s3/S3StorageFetchSaveSpec.scala @@ -48,8 +48,6 @@ class S3StorageFetchSaveSpec default = false, algorithm = DigestAlgorithm.default, bucket = bucket, - endpoint = None, - region = None, readPermission = read, writePermission = write, maxFileSize = 20 diff --git a/docs/src/main/paradox/docs/delta/api/storages-api.md b/docs/src/main/paradox/docs/delta/api/storages-api.md index f620f1301f..3412ef098a 100644 --- a/docs/src/main/paradox/docs/delta/api/storages-api.md +++ b/docs/src/main/paradox/docs/delta/api/storages-api.md @@ -96,8 +96,6 @@ In order to be able to use this storage, the configuration flag `plugins.storage { "@type": "S3Storage", "default": "{default}", - "endpoint": "{endpoint}", - "region": "{region}", "readPermission": "{read_permission}", "writePermission": "{write_permission}", "maxFileSize": {max_file_size} @@ -107,8 +105,6 @@ In order to be able to use this storage, the configuration flag `plugins.storage ...where - `{default}`: Boolean - the flag to decide whether this storage is going to become the default storage for the target project or not. -- `{endpoint}`: Uri - the Amazon S3 compatible service endpoint. This field is optional, defaulting to the configuration flag `plugins.storage.storages.amazon.default-endpoint`. -- `{region}`: String - the Amazon S3 compatible region. This field is optional, defaulting to the S3 default region configuration. - `{read_permission}`: String - the permission a client must have in order to fetch files using this storage. This field is optional, defaulting to the configuration flag `plugins.storage.storages.amazon.default-read-permission` (`resources/read`). - `{write_permission}`: String - the permission a client must have in order to create files using this storage. This field is optional, defaulting to the configuration flag `plugins.storage.storages.amazon.default-write-permission` (`files/write`). - `{max_file_size}`: Long - the maximum allowed size in bytes for files uploaded using this storage. This field is optional, defaulting to the configuration flag `plugins.storage.storages.amazon.default-max-file-size` (10G). diff --git a/tests/src/test/resources/kg/storages/s3-response.json b/tests/src/test/resources/kg/storages/s3-response.json index 39c10537ea..64d5960182 100644 --- a/tests/src/test/resources/kg/storages/s3-response.json +++ b/tests/src/test/resources/kg/storages/s3-response.json @@ -11,7 +11,6 @@ "default": false, "maxFileSize" : {{maxFileSize}}, "bucket": "{{bucket}}", - "endpoint" : "{{endpoint}}", "readPermission": "{{read}}", "writePermission": "{{write}}", "_algorithm": "SHA-256", diff --git a/tests/src/test/resources/kg/storages/s3.json b/tests/src/test/resources/kg/storages/s3.json index 2a8b997437..76d8e6760c 100644 --- a/tests/src/test/resources/kg/storages/s3.json +++ b/tests/src/test/resources/kg/storages/s3.json @@ -2,6 +2,5 @@ "@id": "{{storageId}}", "@type": "S3Storage", "default": false, - "bucket": "{{bucket}}", - "endpoint": "{{endpoint}}" + "bucket": "{{bucket}}" } \ No newline at end of file diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/S3StorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/S3StorageSpec.scala index a42c8f5a68..2ec325b63b 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/S3StorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/S3StorageSpec.scala @@ -77,7 +77,6 @@ class S3StorageSpec extends StorageSpec { "self" -> storageSelf(project, s"https://bluebrain.github.io/nexus/vocabulary/$id"), "bucket" -> bucket, "maxFileSize" -> storageConfig.maxFileSize.toString, - "endpoint" -> s3Endpoint, "read" -> readPermission, "write" -> writePermission ): _* @@ -87,17 +86,14 @@ class S3StorageSpec extends StorageSpec { val payload = jsonContentOf( "kg/storages/s3.json", "storageId" -> s"https://bluebrain.github.io/nexus/vocabulary/$storId", - "bucket" -> bucket, - "endpoint" -> s3Endpoint + "bucket" -> bucket ) val payload2 = jsonContentOf( "kg/storages/s3.json", "storageId" -> s"https://bluebrain.github.io/nexus/vocabulary/${storId}2", - "bucket" -> bucket, - "endpoint" -> s3Endpoint + "bucket" -> bucket ) deepMerge Json.obj( - "region" -> Json.fromString("eu-west-2"), "readPermission" -> Json.fromString(s"$storName/read"), "writePermission" -> Json.fromString(s"$storName/write") ) @@ -106,7 +102,6 @@ class S3StorageSpec extends StorageSpec { val storageId2 = s"${storId}2" val expectedStorageWithPerms = storageResponse(projectRef, storageId2, "s3/read", "s3/write") - .deepMerge(Json.obj("region" -> Json.fromString("eu-west-2"))) for { _ <- storagesDsl.createStorage(payload, projectRef) @@ -122,8 +117,7 @@ class S3StorageSpec extends StorageSpec { val payload = jsonContentOf( "kg/storages/s3.json", "storageId" -> s"https://bluebrain.github.io/nexus/vocabulary/missing", - "bucket" -> "foobar", - "endpoint" -> s3Endpoint + "bucket" -> "foobar" ) deltaClient.post[Json](s"/storages/$projectRef", payload, Coyote) { (json, response) =>