Add remote contexts to Graph Analytics indices (#4297)

BlueBrain · Sep 25, 2023 · 915fe3e · 915fe3e
1 parent 9f12e8b
commit 915fe3e
Show file tree

Hide file tree

Showing 15 changed files with 179 additions and 48 deletions.
diff --git a/delta/plugins/graph-analytics/src/main/resources/elasticsearch/mappings.json b/delta/plugins/graph-analytics/src/main/resources/elasticsearch/mappings.json
@@ -108,6 +108,30 @@
           "type": "keyword"
         }
       }
+    },
+    "remoteContexts": {
+      "type": "nested",
+      "properties": {
+        "iri": {
+          "type": "keyword"
+        },
+        "@type": {
+          "type": "keyword"
+        },
+        "resource": {
+          "properties": {
+            "id": {
+              "type": "keyword"
+            },
+            "project": {
+              "type": "keyword"
+            },
+            "rev": {
+              "type": "long"
+            }
+          }
+        }
+      }
     }
   }
 }
diff --git a/...ch/epfl/bluebrain/nexus/delta/plugins/graph/analytics/indexing/GraphAnalyticsResult.scala b/...ch/epfl/bluebrain/nexus/delta/plugins/graph/analytics/indexing/GraphAnalyticsResult.scala
@@ -3,11 +3,12 @@ package ch.epfl.bluebrain.nexus.delta.plugins.graph.analytics.indexing
 import ch.epfl.bluebrain.nexus.delta.plugins.graph.analytics.model.JsonLdDocument
 import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords
-import io.circe.{Encoder, Json}
+import ch.epfl.bluebrain.nexus.delta.sdk.model.jsonld.RemoteContextRef
 import ch.epfl.bluebrain.nexus.delta.sdk.syntax._
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef
 import io.circe.syntax.{EncoderOps, KeyOps}
+import io.circe.{Encoder, Json}
 
 import java.time.Instant
 
@@ -43,6 +44,7 @@ object GraphAnalyticsResult {
   final case class Index private (
       project: ProjectRef,
       id: Iri,
+      remoteContexts: Set[RemoteContextRef],
       rev: Int,
       deprecated: Boolean,
       types: Set[Iri],
@@ -58,6 +60,7 @@ object GraphAnalyticsResult {
     def active(
         project: ProjectRef,
         id: Iri,
+        remoteContexts: Set[RemoteContextRef],
         rev: Int,
         types: Set[Iri],
         createdAt: Instant,
@@ -66,32 +69,34 @@ object GraphAnalyticsResult {
         updatedBy: Subject,
         value: JsonLdDocument
     ) =
-      new Index(project, id, rev, false, types, createdAt, createdBy, updatedAt, updatedBy, Some(value))
+      new Index(project, id, remoteContexts, rev, false, types, createdAt, createdBy, updatedAt, updatedBy, Some(value))
 
     def deprecated(
         project: ProjectRef,
         id: Iri,
+        remoteContexts: Set[RemoteContextRef],
         rev: Int,
         types: Set[Iri],
         createdAt: Instant,
         createdBy: Subject,
         updatedAt: Instant,
         updatedBy: Subject
     ) =
-      new Index(project, id, rev, true, types, createdAt, createdBy, updatedAt, updatedBy, None)
+      new Index(project, id, remoteContexts, rev, true, types, createdAt, createdBy, updatedAt, updatedBy, None)
 
     implicit val encoder: Encoder[Index] = Encoder.instance { i =>
       import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Database._
       Json
         .obj(
-          keywords.id   := i.id,
-          "_project"    := i.project,
-          "_rev"        := i.rev,
-          "_deprecated" := i.deprecated,
-          "_createdAt"  := i.createdAt,
-          "_createdBy"  := i.createdBy,
-          "_updatedAt"  := i.updatedAt,
-          "_updatedBy"  := i.updatedBy
+          keywords.id      := i.id,
+          "remoteContexts" := i.remoteContexts,
+          "_project"       := i.project,
+          "_rev"           := i.rev,
+          "_deprecated"    := i.deprecated,
+          "_createdAt"     := i.createdAt,
+          "_createdBy"     := i.createdBy,
+          "_updatedAt"     := i.updatedAt,
+          "_updatedBy"     := i.updatedBy
         )
         .addIfNonEmpty(keywords.tpe, i.types)
         .deepMerge(i.value.map(_.asJson).getOrElse(Json.obj()))

diff --git a/...ch/epfl/bluebrain/nexus/delta/plugins/graph/analytics/indexing/GraphAnalyticsStream.scala b/...ch/epfl/bluebrain/nexus/delta/plugins/graph/analytics/indexing/GraphAnalyticsStream.scala
@@ -92,13 +92,10 @@ object GraphAnalyticsStream {
           }
         case Resources.entityType =>
           Task.fromEither(ResourceState.serializer.codec.decodeJson(json)).flatMap {
-            case s if s.deprecated =>
-              Task.pure(
-                Index.deprecated(s.project, s.id, s.rev, s.types, s.createdAt, s.createdBy, s.updatedAt, s.updatedBy)
-              )
-            case s                 =>
-              JsonLdDocument.fromExpanded(s.expanded, findRelationships(project, xas, relationshipBatch)).map { d =>
-                Index.active(s.project, s.id, s.rev, s.types, s.createdAt, s.createdBy, s.updatedAt, s.updatedBy, d)
+            case state if state.deprecated => deprecatedIndex(state)
+            case state                     =>
+              JsonLdDocument.fromExpanded(state.expanded, findRelationships(project, xas, relationshipBatch)).map {
+                doc => activeIndex(state, doc)
               }
           }
         case _                    => Task.pure(Noop)
@@ -107,4 +104,34 @@ object GraphAnalyticsStream {
     StreamingQuery.elems(project, start, SelectFilter.latest, qc, xas, decode)
   }
   // $COVERAGE-ON$
+
+  private def deprecatedIndex(state: ResourceState) =
+    Task.pure(
+      Index.deprecated(
+        state.project,
+        state.id,
+        state.remoteContexts,
+        state.rev,
+        state.types,
+        state.createdAt,
+        state.createdBy,
+        state.updatedAt,
+        state.updatedBy
+      )
+    )
+
+  private def activeIndex(state: ResourceState, doc: JsonLdDocument) =
+    Index.active(
+      state.project,
+      state.id,
+      state.remoteContexts,
+      state.rev,
+      state.types,
+      state.createdAt,
+      state.createdBy,
+      state.updatedAt,
+      state.updatedBy,
+      doc
+    )
+
 }
diff --git a/delta/plugins/graph-analytics/src/test/resources/result/resource1.json b/delta/plugins/graph-analytics/src/test/resources/result/resource1.json
@@ -5,6 +5,12 @@
     "http://www.w3.org/ns/prov#Entity",
     "https://neuroshapes.org/ResponseTrace"
   ],
+  "remoteContexts": [
+    {
+      "iri": "https://bluebrain.github.io/nexus/contexts/metadata.json",
+      "@type": "StaticContextRef"
+    }
+  ],
   "_project": "myorg/myproject",
   "_rev": 1,
   "_deprecated": false,

diff --git a/delta/plugins/graph-analytics/src/test/resources/result/resource1_updated.json b/delta/plugins/graph-analytics/src/test/resources/result/resource1_updated.json
@@ -5,6 +5,12 @@
     "http://www.w3.org/ns/prov#Entity",
     "https://neuroshapes.org/ResponseTrace"
   ],
+  "remoteContexts": [
+    {
+      "iri": "https://bluebrain.github.io/nexus/contexts/metadata.json",
+      "@type": "StaticContextRef"
+    }
+  ],
   "_project": "myorg/myproject",
   "_rev": 1,
   "_deprecated": false,

diff --git a/delta/plugins/graph-analytics/src/test/resources/result/resource2.json b/delta/plugins/graph-analytics/src/test/resources/result/resource2.json
@@ -7,6 +7,12 @@
     "http://schema.org/Dataset",
     "http://www.w3.org/ns/prov#Entity"
   ],
+  "remoteContexts": [
+    {
+      "iri": "https://bluebrain.github.io/nexus/contexts/metadata.json",
+      "@type": "StaticContextRef"
+    }
+  ],
   "_project": "myorg/myproject",
   "_rev": 1,
   "_deprecated": false,

diff --git a/delta/plugins/graph-analytics/src/test/resources/result/resource_deprecated.json b/delta/plugins/graph-analytics/src/test/resources/result/resource_deprecated.json
@@ -4,6 +4,12 @@
     "https://neuroshapes.org/NeuroMorphology"
   ],
   "@id": "http://localhost/deprecated",
+  "remoteContexts": [
+    {
+      "iri": "https://bluebrain.github.io/nexus/contexts/metadata.json",
+      "@type": "StaticContextRef"
+    }
+  ],
   "_project": "myorg/myproject",
   "_rev": 1,
   "_deprecated": true,

diff --git a/...epfl/bluebrain/nexus/delta/plugins/graph/analytics/indexing/GraphAnalyticsSinkSuite.scala b/...epfl/bluebrain/nexus/delta/plugins/graph/analytics/indexing/GraphAnalyticsSinkSuite.scala
@@ -7,6 +7,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.graph.analytics.model.JsonLdDocumen
 import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.nxvFile
 import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri
 import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.ExpandedJsonLd
+import ch.epfl.bluebrain.nexus.delta.sdk.model.jsonld.RemoteContextRef
 import ch.epfl.bluebrain.nexus.delta.sdk.resources.Resources
 import ch.epfl.bluebrain.nexus.delta.sdk.syntax._
 import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Anonymous
@@ -41,6 +42,9 @@ class GraphAnalyticsSinkSuite
 
   private val project = ProjectRef.unsafe("myorg", "myproject")
 
+  private val remoteContexts: Set[RemoteContextRef] =
+    Set(RemoteContextRef.StaticContextRef(iri"https://bluebrain.github.io/nexus/contexts/metadata.json"))
+
   // resource1 has references to 'resource3', 'file1' and 'generatedBy',
   // 'generatedBy' remains unresolved
   private val resource1 = iri"http://localhost/resource1"
@@ -102,13 +106,17 @@ class GraphAnalyticsSinkSuite
         types    <- getTypes(expanded)
         doc      <- JsonLdDocument.fromExpanded(expanded, _ => findRelationships)
       } yield {
-        val result = Index.active(project, id, 1, types, Instant.EPOCH, Anonymous, Instant.EPOCH, Anonymous, doc)
+        val result =
+          Index.active(project, id, remoteContexts, 1, types, Instant.EPOCH, Anonymous, Instant.EPOCH, Anonymous, doc)
         success(id, result)
       }
     }
 
     def indexDeprecated(id: Iri, types: Set[Iri]) =
-      success(id, Index.deprecated(project, id, 1, types, Instant.EPOCH, Anonymous, Instant.EPOCH, Anonymous))
+      success(
+        id,
+        Index.deprecated(project, id, remoteContexts, 1, types, Instant.EPOCH, Anonymous, Instant.EPOCH, Anonymous)
+      )
 
     for {
       active1            <- indexActive(resource1, expanded1)

diff --git a/docs/src/main/paradox/docs/delta/api/assets/graph-analytics/ga-search.json b/docs/src/main/paradox/docs/delta/api/assets/graph-analytics/ga-search.json
@@ -39,6 +39,17 @@
               "path": "http://schema.org/worksFor"
             }
           ],
+          "remoteContexts": [
+            {
+              "@type": "ProjectRemoteContextRef",
+              "iri": "https://bbp.epfl.ch/contexts/person",
+              "resource": {
+                "id": "https://bbp.epfl.ch/contexts/person",
+                "project": "myorg/myproj",
+                "rev": 1
+              }
+            }
+          ],
           "_createdAt": "2023-08-08T15:49:14.081Z",
           "_createdBy": {
             "@type": "User",

diff --git a/docs/src/main/paradox/docs/delta/api/graph-analytics-api.md b/docs/src/main/paradox/docs/delta/api/graph-analytics-api.md
@@ -172,6 +172,17 @@ An example of the ElasticSearch Document looks as follows:
       "path": "http://schema.org/brother",
       "isInArray": false
     }
+  ],
+  "remoteContexts": [
+    {
+      "@type": "ProjectRemoteContextRef",
+      "iri": "https://bbp.epfl.ch/contexts/person",
+      "resource": {
+        "id": "https://bbp.epfl.ch/contexts/person",
+        "project": "myorg/myproject",
+        "rev": 1
+      }
+    }
   ]
 }
 ```
@@ -184,4 +195,6 @@ An example of the ElasticSearch Document looks as follows:
 - `path` - String: The flat expanded path of a field present on a resource. A path of an embedded field will be encoded as follows: `parent / child`.
 - `isInArray` - Boolean: Flag to inform whether the current path (or its parent) is part of an array.
 - `dataType` - String: The type of the value present in the current path. Possible values are: string, numeric and boolean
-- `found` - Boolean: Flag to inform whether an @id inside `references` has been resolved as a relationship.
+- `found` - Boolean: Flag to inform whether an @id inside `references` has been resolved as a relationship.
+- `remoteContexts` - Json Object Array: A collection of remote contexts detected during the JSON-LD resolution for this resource. 
+  See the @ref:[Resources - Fetch remote contexts](resources-api.md#fetch-remote-contexts) operation to learn about the remote context types.
diff --git a/docs/src/main/paradox/docs/releases/v1.9-release-notes.md b/docs/src/main/paradox/docs/releases/v1.9-release-notes.md
@@ -148,12 +148,21 @@ These should instead be defined in the Delta configuration.
 
 @ref:[More information](../getting-started/running-nexus/configuration/index.md#remote-storage-configuration)
 
-### Graph Analytics
+### Graph analytics
 
-The Elasticsearch views behind Graph Analytics can now be queried using the `_search` endpoint.
+#### Search endpoint for Graph analytics' views
+
+The Elasticsearch views behind Graph analytics can now be queried using the `_search` endpoint.
 
 @ref:[More information](../delta/api/graph-analytics-api.md)
 
+#### Indexing remote contexts in Graph Analytics
+
+Remote contexts used during the JSON-LD resolution for a resource are now indexed 
+in the Graph analytics views and can be found via the @ref:[search endpoint](#search-endpoint-for-graph-analytics-views)
+
+@ref:[More information](../delta/api/graph-analytics-api.md#document-structure)
+
 ### Id Resolution
 
 It is now possible to perform ID resolution by providing only the identifier of a resource. 

diff --git a/tests/src/test/resources/kg/graph-analytics/context-test.json b/tests/src/test/resources/kg/graph-analytics/context-test.json
@@ -0,0 +1,13 @@
+{
+  "@id": "https://bbp.epfl.ch/contexts/person",
+  "@context": {
+    "@base": "http://example.com/",
+    "@vocab": "http://schema.org/",
+    "worksFor": {
+      "@type": "@id"
+    },
+    "colleague": {
+      "@type": "@id"
+    }
+  }
+}
diff --git a/tests/src/test/resources/kg/graph-analytics/es-hit-source-person1.json b/tests/src/test/resources/kg/graph-analytics/es-hit-source-person1.json
@@ -63,14 +63,25 @@
       "path": "http://schema.org/colleague"
     }
   ],
+  "remoteContexts": [
+    {
+      "@type": "ProjectRemoteContextRef",
+      "iri": "https://bbp.epfl.ch/contexts/person",
+      "resource": {
+        "id": "https://bbp.epfl.ch/contexts/person",
+        "project": "{{projectRef}}",
+        "rev": 1
+      }
+    }
+  ],
   "_createdAt": "2023-08-08T12:23:16.235Z",
   "_createdBy": {
     "@type": "User",
     "realm": "test-armauidxyrrteqhl",
     "subject": "cynfrbbtjlxrceea"
   },
   "_deprecated": false,
-  "_project": "rft1378gl2ckbd4/m2ufaden6k1k12v",
+  "_project": "{{projectRef}}",
   "_rev": 1,
   "_updatedAt": "2023-08-08T12:23:16.235Z",
   "_updatedBy": {

diff --git a/tests/src/test/resources/kg/graph-analytics/person1.json b/tests/src/test/resources/kg/graph-analytics/person1.json
@@ -1,14 +1,5 @@
 {
-  "@context": {
-    "@base": "http://example.com/",
-    "@vocab": "http://schema.org/",
-    "worksFor": {
-      "@type": "@id"
-    },
-    "colleague": {
-      "@type": "@id"
-    }
-  },
+  "@context": "https://bbp.epfl.ch/contexts/person",
   "@type": "Person",
   "@id": "http://example.com/person1",
   "givenName": "Alice",