From c987e9afcd39cb87af94af1fdf0fb89d14121cf5 Mon Sep 17 00:00:00 2001 From: Daniel Bell Date: Wed, 17 Jan 2024 08:32:33 +0100 Subject: [PATCH] Fix copy tests --- .../utils/ClasspathResourceLoader.scala | 10 +- .../kernel/utils/HandlebarsExpander.scala | 25 +++++ .../plugins/archive/ArchiveRoutesSpec.scala | 2 +- .../files/file-route-metadata-response.json | 5 + .../plugins/storage/files/FileFixtures.scala | 2 +- .../storage/files/batch/BatchCopySuite.scala | 24 ++--- .../storage/files/generators/FileGen.scala | 8 +- .../files/routes/BatchFilesRoutesSpec.scala | 2 +- .../files/routes/FilesRoutesSpec.scala | 91 +++++++++++++------ 9 files changed, 114 insertions(+), 55 deletions(-) create mode 100644 delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/HandlebarsExpander.scala diff --git a/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/ClasspathResourceLoader.scala b/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/ClasspathResourceLoader.scala index 85dfcfc8ed..c53db01558 100644 --- a/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/ClasspathResourceLoader.scala +++ b/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/ClasspathResourceLoader.scala @@ -2,8 +2,7 @@ package ch.epfl.bluebrain.nexus.delta.kernel.utils import cats.effect.{IO, Resource} import ch.epfl.bluebrain.nexus.delta.kernel.utils.ClasspathResourceError.{InvalidJson, InvalidJsonObject, ResourcePathNotFound} -import ch.epfl.bluebrain.nexus.delta.kernel.utils.ClasspathResourceLoader.handleBars -import com.github.jknack.handlebars.{EscapingStrategy, Handlebars} +import ch.epfl.bluebrain.nexus.delta.kernel.utils.ClasspathResourceLoader.handlebarsExpander import fs2.text import io.circe.parser.parse import io.circe.{Json, JsonObject} @@ -54,10 +53,7 @@ class ClasspathResourceLoader private (classLoader: ClassLoader) { resourcePath: String, attributes: (String, Any)* ): IO[String] = { - resourceAsTextFrom(resourcePath).map { - case text if attributes.isEmpty => text - case text => handleBars.compileInline(text).apply(attributes.toMap.asJava) - } + resourceAsTextFrom(resourcePath).map(handlebarsExpander.expand(_, attributes.toMap)) } /** @@ -124,7 +120,7 @@ class ClasspathResourceLoader private (classLoader: ClassLoader) { } object ClasspathResourceLoader { - private[utils] val handleBars = new Handlebars().`with`(EscapingStrategy.NOOP) + private val handlebarsExpander = new HandlebarsExpander /** * Creates a resource loader using the standard ClassLoader diff --git a/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/HandlebarsExpander.scala b/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/HandlebarsExpander.scala new file mode 100644 index 0000000000..9342026658 --- /dev/null +++ b/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/HandlebarsExpander.scala @@ -0,0 +1,25 @@ +package ch.epfl.bluebrain.nexus.delta.kernel.utils + +import com.github.jknack.handlebars.{EscapingStrategy, Handlebars, Helper, Options} + +import scala.jdk.CollectionConverters._ + +class HandlebarsExpander { + + private val handleBars = new Handlebars().`with`(EscapingStrategy.NOOP).registerHelper("empty", new Helper[Iterable[_]] { + override def apply(context: Iterable[_], options: Options): CharSequence = { + context.iterator.isEmpty match { + case true => options.fn() + case false => options.inverse() + } + } + }) + + def expand(templateText: String,attributes: Map[String, Any]) = { + if (attributes.isEmpty) { + templateText + } else { + handleBars.compileInline(templateText).apply(attributes.asJava) + } + } +} diff --git a/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveRoutesSpec.scala b/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveRoutesSpec.scala index 66d80e15fd..be6ea12874 100644 --- a/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveRoutesSpec.scala +++ b/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveRoutesSpec.scala @@ -304,11 +304,11 @@ class ArchiveRoutesSpec extends BaseRouteSpec with StorageFixtures with ArchiveH projectRef, fileId, file.value.attributes, + file.value.userMetadata, storageRef, createdBy = subject, updatedBy = subject ) - .accepted val actualMetadata = result.entryAsJson(s"${project.ref}/compacted/${encode(fileId.toString)}.json") actualMetadata shouldEqual expectedMetadata } diff --git a/delta/plugins/storage/src/test/resources/files/file-route-metadata-response.json b/delta/plugins/storage/src/test/resources/files/file-route-metadata-response.json index b5b7c82e6a..bc0c6f49b3 100644 --- a/delta/plugins/storage/src/test/resources/files/file-route-metadata-response.json +++ b/delta/plugins/storage/src/test/resources/files/file-route-metadata-response.json @@ -13,6 +13,11 @@ "@type": "{{storageType}}", "_rev": {{storageRev}} }, + {{#unless (empty keywords)}} + "keywords": { + {{#each keywords}}"{{@key}}": "{{this}}"{{/each}} + } + {{/unless}}, "_bytes": {{bytes}}, "_digest": { "_value": "{{digest}}", diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala index 2b3adc4479..989dec73ce 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala @@ -54,7 +54,7 @@ trait FileFixtures extends Generators { projRef: ProjectRef = projectRef ): FileAttributes = FileGen.attributes(filename, size, id, projRef, path) - def randomUserMetadata(): FileUserMetadata = FileUserMetadata(Map(Label.unsafe(genString()) -> genString())) + def genUserMetadata(): FileUserMetadata = FileUserMetadata(Map(Label.unsafe(genString()) -> genString())) def entity(filename: String = "file.txt"): MessageEntity = Multipart diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopySuite.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopySuite.scala index 3c096944c2..61e7cf38f0 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopySuite.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopySuite.scala @@ -40,12 +40,12 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit private val sourceFileId = genFileId(sourceProj.ref) private val source = CopyFileSource(sourceProj.ref, NonEmptyList.of(sourceFileId)) private val storageStatEntry = StorageStatEntry(files = 10L, spaceUsed = 5L) - private val stubbedFileAttr = NonEmptyList.of(attributes(genString())) - private val stubbedFileMetadata = NonEmptyList.of(Some(randomUserMetadata())) + private val stubbedFileAttr = attributes(genString()) + private val stubbedFileMetadata = genUserMetadata() test("successfully perform disk copy") { val events = ListBuffer.empty[Event] - val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, diskVal) + val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, diskVal, Some(stubbedFileMetadata)) val (user, aclCheck) = userAuthorizedOnProjectStorage(sourceStorage.value) val batchCopy = mkBatchCopy( @@ -53,13 +53,13 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit fetchStorage = stubbedFetchStorage(sourceStorage, events), aclCheck = aclCheck, stats = stubbedStorageStats(storageStatEntry, events), - diskCopy = stubbedDiskCopy(stubbedFileAttr, events) + diskCopy = stubbedDiskCopy(NonEmptyList.of(stubbedFileAttr), events) ) val destStorage: DiskStorage = genDiskStorage() batchCopy.copyFiles(source, destStorage)(caller(user)).map { obtained => val obtainedEvents = events.toList - assertEquals(obtained, stubbedFileAttr.zip(stubbedFileMetadata)) + assertEquals(obtained, NonEmptyList.of(stubbedFileAttr -> Some(stubbedFileMetadata))) sourceFileWasFetched(obtainedEvents, sourceFileId) sourceStorageWasFetched(obtainedEvents, sourceFileRes.value.storage, sourceProj.ref) destinationDiskStorageStatsWereFetched(obtainedEvents, destStorage) @@ -74,7 +74,7 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit test("successfully perform remote disk copy") { val events = ListBuffer.empty[Event] - val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, remoteVal) + val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, remoteVal, Some(stubbedFileMetadata)) val (user, aclCheck) = userAuthorizedOnProjectStorage(sourceStorage.value) val batchCopy = mkBatchCopy( @@ -82,13 +82,13 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit fetchStorage = stubbedFetchStorage(sourceStorage, events), aclCheck = aclCheck, stats = stubbedStorageStats(storageStatEntry, events), - remoteCopy = stubbedRemoteCopy(stubbedFileAttr, events) + remoteCopy = stubbedRemoteCopy(NonEmptyList.of(stubbedFileAttr), events) ) val destStorage: RemoteDiskStorage = genRemoteStorage() batchCopy.copyFiles(source, destStorage)(caller(user)).map { obtained => val obtainedEvents = events.toList - assertEquals(obtained, stubbedFileAttr.zip(stubbedFileMetadata)) + assertEquals(obtained, NonEmptyList.of(stubbedFileAttr -> Some(stubbedFileMetadata))) sourceFileWasFetched(obtainedEvents, sourceFileId) sourceStorageWasFetched(obtainedEvents, sourceFileRes.value.storage, sourceProj.ref) destinationRemoteStorageStatsWereNotFetched(obtainedEvents) @@ -110,7 +110,7 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit test("fail if a source storage is different to destination storage") { val events = ListBuffer.empty[Event] - val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, diskVal) + val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, diskVal, Some(stubbedFileMetadata)) val (user, aclCheck) = userAuthorizedOnProjectStorage(sourceStorage.value) val batchCopy = mkBatchCopy( @@ -129,7 +129,7 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit test("fail if user does not have read access on a source file's storage") { val events = ListBuffer.empty[Event] - val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, diskVal) + val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, diskVal, Some(stubbedFileMetadata)) val user = genUser() val aclCheck = AclSimpleCheck((user, AclAddress.fromProject(sourceProj.ref), Set())).accepted @@ -148,7 +148,7 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit test("fail if a single source file exceeds max size for destination storage") { val events = ListBuffer.empty[Event] - val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, diskVal, 1000L) + val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, diskVal, Some(stubbedFileMetadata), 1000L) val (user, aclCheck) = userAuthorizedOnProjectStorage(sourceStorage.value) val batchCopy = mkBatchCopy( @@ -172,7 +172,7 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit val capacity = 10L val statEntry = StorageStatEntry(files = 10L, spaceUsed = 1L) val spaceLeft = capacity - statEntry.spaceUsed - val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, diskVal, fileSize) + val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, diskVal, Some(stubbedFileMetadata), fileSize) val (user, aclCheck) = userAuthorizedOnProjectStorage(sourceStorage.value) val batchCopy = mkBatchCopy( diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala index 7d8f1e92c3..dbe230bf80 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala @@ -63,12 +63,13 @@ trait FileGen { self: Generators with FileFixtures => def genOption[A](genA: => A): Option[A] = if (Random.nextInt(2) % 2 == 0) Some(genA) else None def genFileResource(fileId: FileId, context: ProjectContext): FileResource = - genFileResourceWithStorage(fileId, context, genRevision(), 1L) + genFileResourceWithStorage(fileId, context, genRevision(), Some(genUserMetadata()), 1L) def genFileResourceWithStorage( fileId: FileId, context: ProjectContext, storageRef: ResourceRef.Revision, + userMetadata: Option[FileUserMetadata], fileSize: Long ): FileResource = genFileResourceWithIri( @@ -76,18 +77,19 @@ trait FileGen { self: Generators with FileFixtures => fileId.project, storageRef, attributes(genString(), size = fileSize), - Some(randomUserMetadata()) + userMetadata ) def genFileResourceAndStorage( fileId: FileId, context: ProjectContext, storageVal: StorageValue, + userMetadata: Option[FileUserMetadata], fileSize: Long = 1L ): (FileResource, StorageResource) = { val storageRes = StorageGen.resourceFor(genIri(), fileId.project, storageVal) val storageRef = ResourceRef.Revision(storageRes.id, storageRes.id, storageRes.rev) - (genFileResourceWithStorage(fileId, context, storageRef, fileSize), storageRes) + (genFileResourceWithStorage(fileId, context, storageRef, userMetadata, fileSize), storageRes) } def genFileResourceWithIri( diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutesSpec.scala index c79c937e2e..847d41dfa5 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutesSpec.scala @@ -237,6 +237,7 @@ class BatchFilesRoutesSpec extends BaseRouteSpec with StorageFixtures with FileF res.value.project, res.id, res.value.attributes, + res.value.userMetadata, res.value.storage, res.value.storageType, res.rev, @@ -244,7 +245,6 @@ class BatchFilesRoutesSpec extends BaseRouteSpec with StorageFixtures with FileF res.createdBy, res.updatedBy ) - .accepted .mapObject(_.remove("@context")) } diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala index e26c87c4c8..baf9356f52 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala @@ -9,13 +9,13 @@ import akka.http.scaladsl.model.{StatusCodes, Uri} import akka.http.scaladsl.server.Route import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig -import ch.epfl.bluebrain.nexus.delta.kernel.utils.ClasspathResourceLoader +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.ResourcesSearchParams.FileUserMetadata import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.ComputedDigest import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileId, FileRejection} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{contexts => fileContexts, permissions, FileFixtures, Files, FilesConfig} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FileFixtures, Files, FilesConfig, permissions, contexts => fileContexts} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{StorageRejection, StorageStatEntry, StorageType} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{contexts => storageContexts, permissions => storagesPermissions, StorageFixtures, Storages, StoragesConfig, StoragesStatistics} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{StorageFixtures, Storages, StoragesConfig, StoragesStatistics, contexts => storageContexts, permissions => storagesPermissions} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.RdfMediaTypes.`application/ld+json` import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary @@ -39,9 +39,11 @@ import ch.epfl.bluebrain.nexus.delta.sdk.utils.BaseRouteSpec import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.{Anonymous, Authenticated, Group, Subject, User} import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Label, ProjectRef, ResourceRef} +import ch.epfl.bluebrain.nexus.testkit.CirceLiteral import ch.epfl.bluebrain.nexus.testkit.errors.files.FileErrors.{fileAlreadyExistsError, fileIsNotDeprecatedError} import ch.epfl.bluebrain.nexus.testkit.scalatest.ce.CatsIOValues -import io.circe.Json +import io.circe.syntax.EncoderOps +import io.circe.{Json, JsonObject} import org.scalatest._ class FilesRoutesSpec @@ -660,45 +662,74 @@ class FilesRoutesSpec updatedBy: Subject = callerWriter.subject )(implicit baseUri: BaseUri): Json = FilesRoutesSpec - .fileMetadata(project, id, attributes, storage, storageType, rev, deprecated, createdBy, updatedBy) - .accepted + .fileMetadata(project, id, attributes, None, storage, storageType, rev, deprecated, createdBy, updatedBy) private def nxvBase(id: String): String = (nxv + id).toString } -object FilesRoutesSpec { - private val loader = ClasspathResourceLoader() +object FilesRoutesSpec extends CirceLiteral { def fileMetadata( project: ProjectRef, id: Iri, attributes: FileAttributes, + userMetadata: Option[FileUserMetadata], storage: ResourceRef.Revision, storageType: StorageType = StorageType.DiskStorage, rev: Int = 1, deprecated: Boolean = false, createdBy: Subject, updatedBy: Subject - )(implicit baseUri: BaseUri): IO[Json] = - loader.jsonContentOf( - "files/file-route-metadata-response.json", - "project" -> project, - "id" -> id, - "rev" -> rev, - "storage" -> storage.iri, - "storageType" -> storageType, - "storageRev" -> storage.rev, - "bytes" -> attributes.bytes, - "digest" -> attributes.digest.asInstanceOf[ComputedDigest].value, - "algorithm" -> attributes.digest.asInstanceOf[ComputedDigest].algorithm, - "filename" -> attributes.filename, - "mediaType" -> attributes.mediaType.fold("")(_.value), - "origin" -> attributes.origin, - "uuid" -> attributes.uuid, - "deprecated" -> deprecated, - "createdBy" -> createdBy.asIri, - "updatedBy" -> updatedBy.asIri, - "type" -> storageType, - "self" -> ResourceUris("files", project, id).accessUri - ) + )(implicit baseUri: BaseUri): Json = { + val self = ResourceUris("files", project, id).accessUri + val keywordsJson: Json = userMetadata match { + case Some(meta) => + Json.obj( + "keywords" -> JsonObject.fromIterable( + meta.keywords.map { + case (k, v) => k.value -> v.asJson + } + ).toJson + ) + case None => Json.obj() + } + + val mainJson = json""" + { + "@context" : [ + "https://bluebrain.github.io/nexus/contexts/files.json", + "https://bluebrain.github.io/nexus/contexts/metadata.json" + ], + "@id" : "$id", + "@type" : "File", + "_constrainedBy" : "https://bluebrain.github.io/nexus/schemas/files.json", + "_createdAt" : "1970-01-01T00:00:00Z", + "_createdBy" : "${createdBy.asIri}", + "_storage" : { + "@id": "${storage.iri}", + "@type": "$storageType", + "_rev": ${storage.rev} + }, + "_bytes": ${attributes.bytes}, + "_digest": { + "_value": "${attributes.digest.asInstanceOf[ComputedDigest].value}", + "_algorithm": "${attributes.digest.asInstanceOf[ComputedDigest].algorithm}" + }, + "_filename": "${attributes.filename}", + "_origin": "${attributes.origin}", + "_mediaType": "${attributes.mediaType.fold("")(_.value)}", + "_deprecated" : $deprecated, + "_incoming" : "$self/incoming", + "_outgoing" : "$self/outgoing", + "_project" : "http://localhost/v1/projects/$project", + "_rev" : $rev, + "_self" : "$self", + "_updatedAt" : "1970-01-01T00:00:00Z", + "_updatedBy" : "${updatedBy.asIri}", + "_uuid": "${attributes.uuid}" + } + """ + + mainJson deepMerge(keywordsJson) + } }