Fix annotated source format (#4747)

* Fix annotated source format --------- Co-authored-by: Simon Dumas <[email protected]>
BlueBrain · Feb 21, 2024 · da8b86a · da8b86a
1 parent 58e1bbf
commit da8b86a
Show file tree

Hide file tree

Showing 12 changed files with 144 additions and 70 deletions.
diff --git a/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/routes/ResourcesRoutes.scala b/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/routes/ResourcesRoutes.scala
@@ -4,7 +4,6 @@ import akka.http.scaladsl.model.StatusCodes.{Created, OK}
 import akka.http.scaladsl.server._
 import cats.effect.IO
 import cats.syntax.all._
-import ch.epfl.bluebrain.nexus.delta.rdf.RdfError
 import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.schemas
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution}
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.encoder.JsonLdEncoder
@@ -210,7 +209,7 @@ final class ResourcesRoutes(
                                 emit(
                                   resources
                                     .fetch(resourceRef, project, schemaOpt)
-                                    .flatMap(asSourceWithMetadata)
+                                    .map(asSourceWithMetadata)
                                     .attemptNarrow[ResourceRejection]
                                 )
                               } else {
@@ -320,9 +319,7 @@ object ResourcesRoutes {
 
   def asSourceWithMetadata(
       resource: ResourceF[Resource]
-  )(implicit baseUri: BaseUri, cr: RemoteContextResolution): IO[Json] =
-    AnnotatedSource(resource, resource.value.source).adaptError { case e: RdfError =>
-      InvalidJsonLdFormat(Some(resource.id), e)
-    }
+  )(implicit baseUri: BaseUri): Json =
+    AnnotatedSource(resource, resource.value.source)
 
 }
diff --git a/delta/app/src/test/resources/multi-fetch/annotated-source-response.json b/delta/app/src/test/resources/multi-fetch/annotated-source-response.json
@@ -5,9 +5,14 @@
       "@id": "https://bluebrain.github.io/nexus/vocabulary/success",
       "project": "org/proj1",
       "value": {
-        "@context": "https://bluebrain.github.io/nexus/contexts/metadata.json",
+        "@context": [
+          {
+            "@vocab": "https://bluebrain.github.io/nexus/vocabulary/"
+          },
+          "https://bluebrain.github.io/nexus/contexts/metadata.json"
+        ],
         "@id": "https://bluebrain.github.io/nexus/vocabulary/success",
-        "@type": "https://bluebrain.github.io/nexus/vocabulary/Custom",
+        "@type": "Custom",
         "bool": false,
         "name": "Alex",
         "number": 24,

diff --git a/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/ResourcesRoutesSpec.scala b/delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/ResourcesRoutesSpec.scala
@@ -613,20 +613,18 @@ class ResourcesRoutesSpec extends BaseRouteSpec with CatsIOValues {
     }
 
     "fetch a resource original payload with metadata using tags and revisions" in {
-      val mySchema = "myschema"
+      val mySchema = UrlUtils.encode(schemas.resources.toString)
       val myTag    = "myTag"
-      givenAResourceWithSchemaAndTag(mySchema.some, myTag.some) { id =>
+      givenAResourceWithTag(myTag) { id =>
         val endpoints = List(
           s"/v1/resources/myorg/myproject/$mySchema/$id/source?rev=1&annotate=true",
           s"/v1/resources/myorg/myproject/_/$id/source?rev=1&annotate=true",
           s"/v1/resources/myorg/myproject/$mySchema/$id/source?tag=$myTag&annotate=true"
         )
-        val meta      = standardWriterMetadata(id, schema = schema1.id)
-
         forAll(endpoints) { endpoint =>
           Get(endpoint) ~> asReader ~> routes ~> check {
             status shouldEqual StatusCodes.OK
-            response.asJson shouldEqual simplePayload(id).deepMerge(meta)
+            response.asJson shouldEqual payloadWithMetadata(id)
             response.headers should contain(varyHeader)
           }
         }

diff --git a/...rchive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownload.scala b/...rchive/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownload.scala
@@ -242,9 +242,8 @@ object ArchiveDownload {
         repr match {
           case SourceJson          => IO.pure(ByteString(prettyPrintSource(value.source)))
           case AnnotatedSourceJson =>
-            AnnotatedSource(value.resource, value.source).map { json =>
-              ByteString(prettyPrintSource(json))
-            }
+            val annotatedSource = AnnotatedSource(value.resource, value.source)
+            IO.pure(ByteString(prettyPrintSource(annotatedSource)))
           case CompactedJsonLd     => value.resource.toCompactedJsonLd.map(v => ByteString(prettyPrint(v.json)))
           case ExpandedJsonLd      => value.resource.toExpandedJsonLd.map(v => ByteString(prettyPrint(v.json)))
           case NTriples            => value.resource.toNTriples.map(v => ByteString(v.value))

diff --git a/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/syntax/JsonSyntax.scala b/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/syntax/JsonSyntax.scala
@@ -243,6 +243,11 @@ final class JsonOps(private val json: Json) extends AnyVal {
     */
   def removeAllKeys(keys: String*): Json = JsonUtils.removeAllKeys(json, keys: _*)
 
+  /**
+    * Removes the metadata keys from the current json.
+    */
+  def removeMetadataKeys: Json = JsonUtils.removeMetadataKeys(json)
+
   /**
     * Removes the provided key value pairs from everywhere on the json.
     */

diff --git a/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/utils/JsonUtils.scala b/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/utils/JsonUtils.scala
@@ -48,6 +48,17 @@ trait JsonUtils {
     )
   }
 
+  /**
+    * Remove metadata keys (starting with `_`) from the json
+    */
+  def removeMetadataKeys(json: Json): Json = {
+    json.arrayOrObject(
+      json,
+      arr => Json.fromValues(arr),
+      obj => Json.fromJsonObject(obj.filterKeys(!_.startsWith("_")))
+    )
+  }
+
   /**
     * Extract all the values found from the passed ''keys''
     *

diff --git a/delta/rdf/src/test/scala/ch/epfl/bluebrain/nexus/delta/rdf/utils/JsonUtilsSpec.scala b/delta/rdf/src/test/scala/ch/epfl/bluebrain/nexus/delta/rdf/utils/JsonUtilsSpec.scala
@@ -169,6 +169,12 @@ class JsonUtilsSpec extends BaseSpec with Fixtures {
       jobj"""{"k": "v"}""".addIfExists("k2", Some("v2")) shouldEqual jobj"""{"k": "v", "k2": "v2"}"""
       jobj"""{"k": "v"}""".addIfExists[String]("k2", None) shouldEqual jobj"""{"k": "v"}"""
     }
+
+    "remove metadata keys" in {
+      val json     = json"""{ "k1": "v1",  "k2": { "_nested": "v2" }, "_m1": "v3", "_m2": "v4" }"""
+      val expected = json"""{ "k1": "v1",  "k2": { "_nested": "v2" } }"""
+      JsonUtils.removeMetadataKeys(json) shouldEqual expected
+    }
   }
 
   "A Json cursor" should {

diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/marshalling/AnnotatedSource.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/marshalling/AnnotatedSource.scala
@@ -1,42 +1,21 @@
 package ch.epfl.bluebrain.nexus.delta.sdk.marshalling
 
-import cats.effect.IO
 import cats.syntax.all._
-import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.{JsonLdApi, JsonLdJavaApi}
-import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.RemoteContextResolution
-import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.encoder.JsonLdEncoder
+import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.contexts
 import ch.epfl.bluebrain.nexus.delta.sdk.model.{BaseUri, ResourceF}
+import ch.epfl.bluebrain.nexus.delta.sdk.syntax._
 import io.circe.Json
+import io.circe.syntax.EncoderOps
 
 object AnnotatedSource {
 
-  implicit private val api: JsonLdApi = JsonLdJavaApi.lenient
-
   /**
     * Merges the source with the metadata of [[ResourceF]]
     */
-  def apply(resourceF: ResourceF[_], source: Json)(implicit
-      baseUri: BaseUri,
-      cr: RemoteContextResolution
-  ): IO[Json] =
-    metadataJson(resourceF)
-      .map(mergeOriginalPayloadWithMetadata(source, _))
-
-  private def metadataJson(resource: ResourceF[_])(implicit baseUri: BaseUri, cr: RemoteContextResolution) = {
-    implicit val resourceFJsonLdEncoder: JsonLdEncoder[ResourceF[Unit]] = ResourceF.defaultResourceFAJsonLdEncoder
-    resourceFJsonLdEncoder
-      .compact(resource.void)
-      .map(_.json)
-  }
-
-  private def mergeOriginalPayloadWithMetadata(payload: Json, metadata: Json): Json = {
-    getId(payload)
-      .foldLeft(payload.deepMerge(metadata))(setId)
+  def apply(resourceF: ResourceF[_], source: Json)(implicit baseUri: BaseUri): Json = {
+    val sourceWithoutMetadata = source.removeMetadataKeys
+    val metadataJson          = resourceF.void.asJson
+    metadataJson.deepMerge(sourceWithoutMetadata).addContext(contexts.metadata)
   }
 
-  private def getId(payload: Json): Option[String] = payload.hcursor.get[String]("@id").toOption
-
-  private def setId(payload: Json, id: String): Json =
-    payload.hcursor.downField("@id").set(Json.fromString(id)).top.getOrElse(payload)
-
 }
diff --git a/...rc/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/multifetch/model/MultiFetchResponse.scala b/...rc/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/multifetch/model/MultiFetchResponse.scala
@@ -93,7 +93,7 @@ object MultiFetchResponse {
           val source                             = content.source
           repr match {
             case SourceJson          => IO.pure(source.asJson)
-            case AnnotatedSourceJson => AnnotatedSource(value, source)
+            case AnnotatedSourceJson => IO.pure(AnnotatedSource(value, source))
             case CompactedJsonLd     => value.toCompactedJsonLd.map { v => v.json }
             case ExpandedJsonLd      => value.toExpandedJsonLd.map { v => v.json }
             case NTriples            => value.toNTriples.map { v => v.value.asJson }

diff --git a/delta/sdk/src/test/resources/resources/resource-with-metadata.json b/delta/sdk/src/test/resources/resources/resource-with-metadata.json
@@ -1,7 +1,12 @@
 {
-  "@context": "https://bluebrain.github.io/nexus/contexts/metadata.json",
+  "@context": [
+    {
+      "@vocab": "https://bluebrain.github.io/nexus/vocabulary/"
+    },
+    "https://bluebrain.github.io/nexus/contexts/metadata.json"
+  ],
   "@id": "{{id}}",
-  "@type": "https://bluebrain.github.io/nexus/vocabulary/Custom",
+  "@type": "Custom",
   "bool": false,
   "name": "Alex",
   "number": 24,

diff --git a/...k/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/marshalling/AnnotatedSourceSuite.scala b/...k/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/marshalling/AnnotatedSourceSuite.scala
@@ -1,24 +1,22 @@
 package ch.epfl.bluebrain.nexus.delta.sdk.marshalling
 
-import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri
 import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.{contexts, nxv, schemas}
-import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution}
+import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.ContextValue
+import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.ContextValue.ContextRemoteIri
 import ch.epfl.bluebrain.nexus.delta.sdk.model.{BaseUri, ResourceF, ResourceUris}
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Anonymous
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.ResourceRef.Latest
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Label, ProjectRef}
 import ch.epfl.bluebrain.nexus.testkit.CirceLiteral
 import ch.epfl.bluebrain.nexus.testkit.mu.NexusSuite
+import io.circe.syntax.{EncoderOps, KeyOps}
+import io.circe.{Json, JsonObject}
+import munit.Location
 
 import java.time.Instant
 
 class AnnotatedSourceSuite extends NexusSuite with CirceLiteral {
 
-  implicit private def res: RemoteContextResolution =
-    RemoteContextResolution.fixedIO(
-      contexts.metadata -> ContextValue.fromFile("contexts/metadata.json")
-    )
-
   implicit val baseUri: BaseUri = BaseUri("http://localhost", Label.unsafe("v1"))
 
   private val id       = nxv + "id"
@@ -37,9 +35,8 @@ class AnnotatedSourceSuite extends NexusSuite with CirceLiteral {
     ()
   )
 
-  private def expected(expectedId: Iri) =
-    json"""{
-             "@context" : "https://bluebrain.github.io/nexus/contexts/metadata.json",
+  private val metadataJson =
+    jobj"""{
              "_constrainedBy" : "https://bluebrain.github.io/nexus/schemas/unconstrained.json",
              "_createdAt" : "1970-01-01T00:00:00Z",
              "_createdBy" : "http://localhost/v1/anonymous",
@@ -51,21 +48,87 @@ class AnnotatedSourceSuite extends NexusSuite with CirceLiteral {
              "_schemaProject" : "http://localhost/v1/projects/org/proj",
              "_self" : "http://localhost/v1/resources/org/proj/_/https:%2F%2Fbluebrain.github.io%2Fnexus%2Fschemas%2Funconstrained.json",
              "_updatedAt" : "1970-01-01T00:00:00Z",
-             "_updatedBy" : "http://localhost/v1/anonymous",
-             "@id" : "$expectedId",
-             "@type" : "https://bluebrain.github.io/nexus/vocabulary/Type",
-             "source" : "original payload"
+             "_updatedBy" : "http://localhost/v1/anonymous"
            }"""
 
-  test("Merge metadata and source injecting the missing id from the source") {
-    val source = json"""{"source": "original payload"}"""
-    AnnotatedSource(resource, source).assertEquals(expected(id))
+  private def assertResult(
+      result: Json,
+      expectedId: String,
+      expectedType: String,
+      expectedContext: ContextValue,
+      payloadFields: (String, Json)*
+  )(implicit l: Location) = {
+    def onObject(obj: JsonObject) = {
+      assertEquals(obj("@id"), Some(expectedId.asJson))
+      assertEquals(obj("@type"), Some(expectedType.asJson))
+      assertEquals(obj("@context"), Some(expectedContext.asJson))
+      val obtainedMetadata = obj.filterKeys(_.startsWith("_"))
+      assertEquals(obtainedMetadata, metadataJson)
+      val payloadData      = obj.filterKeys { k => !k.startsWith("_") && !k.startsWith("@") }
+      assertEquals(payloadData, JsonObject(payloadFields: _*))
+    }
+
+    result.arrayOrObject(
+      fail("We expected an object, we got a literal"),
+      _ => fail("We expected an object, we got an array"),
+      onObject
+    )
+  }
+
+  test("Merge metadata and source for a resource without an id, type or context") {
+    val source = json"""{"source": "original payload" }"""
+    assertResult(
+      AnnotatedSource(resource, source),
+      id.toString,
+      resource.types.mkString,
+      ContextRemoteIri(contexts.metadata),
+      "source" := "original payload"
+    )
+  }
+
+  test("Exclude invalid metadata at the root level") {
+    val source = json"""{"source": "original payload", "_rev": 42, "_other": "xxx", "nested": { "_rev": 5} }"""
+    assertResult(
+      AnnotatedSource(resource, source),
+      id.toString,
+      resource.types.mkString,
+      ContextRemoteIri(contexts.metadata),
+      "source" := "original payload",
+      "nested" := JsonObject("_rev" := 5)
+    )
   }
 
-  test("Merge metadata and source keeping the id from the source") {
-    val sourceId = nxv + "sourceId"
-    val source   = json"""{ "@id": "$sourceId", "source": "original payload"}"""
-    AnnotatedSource(resource, source).assertEquals(expected(sourceId))
+  test("Merge metadata and source with an id and a type but no context") {
+    val sourceId   = "id"
+    val sourceType = "Type"
+    val source     = json"""{ "@id": "$sourceId", "@type": "$sourceType", "source": "original payload" }"""
+    assertResult(
+      AnnotatedSource(resource, source),
+      "id",
+      "Type",
+      ContextRemoteIri(contexts.metadata),
+      "source" := "original payload"
+    )
+  }
+
+  test("Merge metadata and source with an id, a type and a context") {
+    val sourceId      = "id"
+    val sourceType    = "Type"
+    val sourceContext = nxv + "context"
+    val source        =
+      json"""{
+            "@context" : "$sourceContext",
+            "@id": "$sourceId",
+            "@type": "$sourceType",
+            "source": "original payload"
+      }"""
+    assertResult(
+      AnnotatedSource(resource, source),
+      "id",
+      "Type",
+      ContextValue(sourceContext, contexts.metadata),
+      "source" := "original payload"
+    )
   }
 
 }
diff --git a/tests/src/test/resources/kg/resources/simple-resource-with-metadata.json b/tests/src/test/resources/kg/resources/simple-resource-with-metadata.json
@@ -1,7 +1,13 @@
 {
-  "@context": "https://bluebrain.github.io/nexus/contexts/metadata.json",
+  "@context": [
+    {
+      "nxv": "https://bluebrain.github.io/nexus/vocabulary/",
+      "other": "https://some.other.prefix.com/"
+    },
+    "https://bluebrain.github.io/nexus/contexts/metadata.json"
+  ],
   "@id": "{{resourceId}}",
-  "@type": "https://bluebrain.github.io/nexus/vocabulary/TestResource",
+  "@type": "nxv:TestResource",
   "_constrainedBy": "https://dev.nexus.test.com/test-schema",
   "_createdBy": "{{deltaUri}}/realms/{{realm}}/users/{{user}}",
   "_updatedBy": "{{deltaUri}}/realms/{{realm}}/users/{{user}}",