diff --git a/contexts/aggregations.json b/contexts/aggregations.json new file mode 100644 index 0000000000..1f8226c417 --- /dev/null +++ b/contexts/aggregations.json @@ -0,0 +1,9 @@ +{ + "@context": { + "@vocab": "https://bluebrain.github.io/nexus/vocabulary/" + }, + "@id": "https://bluebrain.github.io/nexus/contexts/aggregations.json", + "aggregations": { + "@container": "@list" + } +} \ No newline at end of file diff --git a/contexts/composite-views.json b/contexts/composite-views.json index cbf0e46e0f..4ffb0bd1a4 100644 --- a/contexts/composite-views.json +++ b/contexts/composite-views.json @@ -15,10 +15,10 @@ "@container": "@set" }, "sources": { - "@container": "@set" + "@container": "@list" }, "projections": { - "@container": "@set" + "@container": "@list" }, "_uuid": "https://bluebrain.github.io/nexus/vocabulary/uuid" }, diff --git a/contexts/elasticsearch-metadata.json b/contexts/elasticsearch-metadata.json index 7bbc67f7b9..a344a8863a 100644 --- a/contexts/elasticsearch-metadata.json +++ b/contexts/elasticsearch-metadata.json @@ -3,8 +3,7 @@ "View": "https://bluebrain.github.io/nexus/vocabulary/View", "ElasticSearchView": "https://bluebrain.github.io/nexus/vocabulary/ElasticSearchView", "AggregateElasticSearchView": "https://bluebrain.github.io/nexus/vocabulary/AggregateElasticSearchView", - "_uuid": "https://bluebrain.github.io/nexus/vocabulary/uuid", - "_indexingRev": "https://bluebrain.github.io/nexus/vocabulary/indexingRev" + "_uuid": "https://bluebrain.github.io/nexus/vocabulary/uuid" }, "@id": "https://bluebrain.github.io/nexus/contexts/elasticsearch-metadata.json" } diff --git a/contexts/elasticsearch.json b/contexts/elasticsearch.json index 8b377b72e4..8a27561e57 100644 --- a/contexts/elasticsearch.json +++ b/contexts/elasticsearch.json @@ -18,8 +18,7 @@ "viewId": { "@type": "@id" }, - "_uuid": "https://bluebrain.github.io/nexus/vocabulary/uuid", - "_indexingRev": "https://bluebrain.github.io/nexus/vocabulary/indexingRev" + "_uuid": "https://bluebrain.github.io/nexus/vocabulary/uuid" }, "https://bluebrain.github.io/nexus/contexts/pipeline.json" ], diff --git a/contexts/metadata.json b/contexts/metadata.json index e3193c6aad..f76fa81274 100644 --- a/contexts/metadata.json +++ b/contexts/metadata.json @@ -1,6 +1,10 @@ { "@context": { "_rev": "https://bluebrain.github.io/nexus/vocabulary/rev", + "_tags": { + "@id": "https://bluebrain.github.io/nexus/vocabulary/tags", + "@container": "@set" + }, "_deprecated": "https://bluebrain.github.io/nexus/vocabulary/deprecated", "_createdAt": { "@id": "https://bluebrain.github.io/nexus/vocabulary/createdAt", diff --git a/contexts/remote-contexts.json b/contexts/remote-contexts.json new file mode 100644 index 0000000000..641de69e4f --- /dev/null +++ b/contexts/remote-contexts.json @@ -0,0 +1,9 @@ +{ + "@context": { + "@vocab": "https://bluebrain.github.io/nexus/vocabulary/", + "remoteContexts": { + "@container": "@set" + } + }, + "@id": "https://bluebrain.github.io/nexus/contexts/remote-contexts.json" +} \ No newline at end of file diff --git a/contexts/resolvers.json b/contexts/resolvers.json index ed45ffd5cb..c2dd3b633a 100644 --- a/contexts/resolvers.json +++ b/contexts/resolvers.json @@ -13,6 +13,12 @@ "identities": { "@container": "@set" }, + "history": { + "@container": "@list" + }, + "rejections": { + "@container": "@list" + }, "subject": "https://bluebrain.github.io/nexus/vocabulary/subject", "_resolverId": { "@id": "https://bluebrain.github.io/nexus/vocabulary/resolverId", diff --git a/docs/delta/api/acls-api.html b/docs/delta/api/acls-api.html index 260b3faaea..27579ff984 100644 --- a/docs/delta/api/acls-api.html +++ b/docs/delta/api/acls-api.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
  • Fetch resource using resolvers
  • +
  • Fetch original resource payload using resolvers
  • Server Sent Events
  • @@ -319,6 +321,13 @@
    +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    Resolvers

    Resolvers are rooted in the /v1/resolvers/{org_label}/{project_label} collection and are used in the following scenarios:

    @@ -814,6 +823,7 @@

    Within an organization

    This operation returns only resolvers from projects defined in the organisation {org_label} and where the caller has the resources/read permission.

    @@ -826,6 +836,7 @@

    Within all projects

    This operation returns only resolvers from projects where the caller has the resources/read permission.

    @@ -838,6 +849,7 @@

    Parameter description

    Example

    @@ -955,6 +968,31 @@

    Fetch original resource payload using resolvers

    +

    Fetches the original source payload of a resource using the provided resolver.

    +

    If the resolver segment ({resolver_id}) is _ the resource is fetched from the first resolver in the requested project ({org_label}/{project_label}). The resolvers are ordered by its priority field.

    +
    GET /v1/resolvers/{org_label}/{project_label}/{resolver_id}/{resource_id}/source?rev={rev}&tag={tag}
    +
    +

    where … - {resource_id}: Iri - the @id value of the resource to be retrieved. - {rev}: Number - the targeted revision to be fetched. This field is optional and defaults to the latest revision. - {tag}: String - the targeted tag to be fetched. This field is optional.

    +

    {rev} and {tag} fields cannot be simultaneously present.

    +

    Example

    +
    +
    Request
    +
    +
    sourcecurl "http://localhost:8080/v1/resolvers/myorg/myproj/nxv:myresolver/fd8a2b32-170e-44e8-808f-44a8cbbc49b0/source"
    +
    Response
    +
    +
    source{
    +  "@context": {
    +    "ex": "http://localhost:8080/",
    +    "@vocab": "http://localhost:8080/"
    +  },
    +  "@type": "ex:Custom",
    +  "name": "Alex",
    +  "number": 24,
    +  "bool": false
    +}
    +

    Server Sent Events

    From Delta 1.5, it is possible to fetch SSEs for all resolvers or just resolvers in the scope of an organization or a project.

    GET /v1/resolvers/events # for all resolver events in the application
    @@ -999,7 +1037,7 @@ 

    @@ -1009,7 +1047,7 @@

    diff --git a/docs/delta/api/resources-api.html b/docs/delta/api/resources-api.html index b9e8cedc88..58d1da5f02 100644 --- a/docs/delta/api/resources-api.html +++ b/docs/delta/api/resources-api.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +
    Note
    -

    From Delta v1.5, remote contexts are only resolved during creates and updates. That means that when those get updated, the resources importing them must be also updated to take them into account the new version.

    +

    Please visit Authentication & authorization section to learn more about it.

    Remote contexts
    +

    Remote contexts are only resolved during creates and updates. That means that when those get updated, the resources importing them must be also updated to take them into account in a new version.

    JSON payloads
    +

    The json payload for create and update operations cannot contain keys beginning with underscore (_), as these fields are reserved for Nexus metadata

    +

    Nexus metadata

    +

    When using the endpoints described on this page, the responses will contain global metadata described on the Nexus Metadata page. In addition, the following resource specific metadata can be present

    +
      +
    • _project: address of the resource’s project
    • +
    • _incoming: address to query to obtain the list of incoming links
    • +
    • _outgoing: address to query to obtain the list of outgoing links
    • +
    • _constrainedBy: @id of the schema used to validate the resource; the schema can only be identified uniquely together with _schemaProject. If no schema has been used to validate the resource, it will indicate the unconstrained identifier.
    • +
    • _schemaProject: address of the project where the _constrainedBy schema is found
    • +

    Indexing

    All the API calls modifying a resource (creation, update, tagging, deprecation) can specify whether the resource should be indexed synchronously or in the background. This behaviour is controlled using indexing query param, which can be one of two values:

    Create using POST

    -
    POST /v1/resources/{org_label}/{project_label}/{schema_id}
    +
    POST /v1/resources/{org_label}/{project_label}/{schema_id}?tag={tag}
       {...}
     

    The json payload:

    @@ -355,6 +380,12 @@

    If the @id value is found on the payload, this @id will be used.
  • If the @id value is not found on the payload, an @id will be generated as follows: base:{UUID}. The base is the prefix defined on the resource’s project ({project_label}).
  • +

    The {schema_id} segment allows to define an existing SHACL schema to validate the resource with:

    +
      +
    • If _ is provided, no SHACL validation will be performed
    • +
    • If another value is provided, Nexus will attempt to resolve the schema then validate the expanded JSON-LD value generated from the provided payload.
    • +
    +

    The {tag} param is an optional tag associated with the first revision of this resource

    Example

    Request
    @@ -407,9 +438,11 @@

    Create using PUT

    This alternative endpoint to create a resource is useful in case the json payload does not contain an @id but you want to specify one. The @id will be specified in the last segment of the endpoint URI.

    -
    PUT /v1/resources/{org_label}/{project_label}/{schema_id}/{resource_id}
    +
    PUT /v1/resources/{org_label}/{project_label}/{schema_id}/{resource_id}?tag={tag}
       {...}
     
    +

    … where {tag} is an optional tag associated with the first revision of this resource.

    +

    The {schema_id} has the same behaviour as the creation using post operation.

    Note that if the payload contains an @id different from the {resource_id}, the request will fail.

    Example

    @@ -461,13 +494,24 @@

    -

    Update

    +

    Update

    A update operation which does not result in a change of the existing resources is ignored
    +

    If the submitted update does not result in changes compared to the current revision of the resource, it will be ignored and the current revision will be returned to the user.

    +

    Nevertheless, if a tag is provided, it will still be applied to the current revision of the resource.

    This operation overrides the payload.

    In order to ensure a client does not perform any changes to a resource without having had seen the previous revision of the resource, the last revision needs to be passed as a query parameter.

    -
    PUT /v1/resources/{org_label}/{project_label}/{schema_id}/{resource_id}?rev={previous_rev}
    +
    PUT /v1/resources/{org_label}/{project_label}/{schema_id}/{resource_id}?rev={previous_rev}&tag={tag}
       {...}
     
    -

    … where {previous_rev} is the last known revision number for the resource.

    +

    … where

    +
      +
    • {previous_rev} is the last known revision number for the resource.
    • +
    • {tag} is an optional tag associated with the same revision as the current update. For example, if previous_rev is 2, both the updated payload and tag will be associated with revision 3; the new latest revision.
    • +
    +

    The {schema_id} segment allows to define an existing SHACL schema to validate the resource with:

    +
      +
    • If _ is provided, no SHACL validation will be performed with the latest version of its current schema
    • +
    • If another value is provided, it has to match the identifier of the current schema as changing the schema of a resource is not currently supported. A different revision or tag of this schema can be provided though.
    • +

    Example

    Request
    @@ -518,7 +562,11 @@

    -

    Refresh

    +

    Refresh

    A refresh operation which does not result in a change of the existing resources is ignored
    +

    If the refresh does not result in changes compared to the current revision of the resource, the update will be ignored and the current revision will be returned to the user.

    +

    If the submitted update does not result in changes compared to the current revision of the resource, it will be ignored and the current revision will be returned to the user.

    +

    Nevertheless, if a tag is provided, it will still be applied to the current revision of the resource.

    +

    @@@

    This operation refreshes the compacted and expanded representations of the resource.

    This is equivalent of doing an update with the same source as the last revision of the resource. It is useful when the schema or project contexts have changed, in order for the changes to be reflected in the resource.

    PUT /v1/resources/{org_label}/{project_label}/{schema_id}/{resource_id}/refresh
    @@ -549,24 +597,6 @@ 

    < "_updatedBy": "http://localhost:8080/v1/realms/myrealm/users/john" }

    -

    Validate

    -

    This operation runs validation of a resource against a schema. This would be useful to test whether resources would match the shape of a new schema.

    -
    GET /v1/resources/{org_label}/{project_label}/{schema_id}/{resource_id}/validate
    -
    -

    Example

    -
    -
    Request
    -
    -
    sourcecurl "http://localhost:8080/v1/resources/myorg/myproj/myschema/base:fd8a2b32-170e-44e8-808f-44a8cbbc49b0/validate"
    -
    Response
    -
    -
    source{
    -  "@context" : "https://bluebrain.github.io/nexus/contexts/shacl-20170720.json",
    -  "@type" : "sh:ValidationReport",
    -  "conforms" : true,
    -  "targetedNodes" : 2
    -}
    -

    Tag

    Links a resource revision to a specific name.

    Tagging a resource is considered to be an update as well.

    @@ -689,6 +719,69 @@

    Undeprecate

    +

    Unlocks a previously deprecated resource. Further operations can then be performed. The resource will again be found when listing/querying.

    +

    Undeprecating a resource is considered to be an update as well.

    +
    PUT /v1/resources/{org_label}/{project_label}/{schema_id}/{resource_id}/undeprecate?rev={previous_rev}
    +
    +

    … where {previous_rev} is the last known revision number for the resource.

    +

    Example

    +
    +
    Request
    +
    +
    sourcecurl -X PUT \
    +     -H "Content-Type: application/json" \
    +     "http://localhost:8080/v1/resources/myorg/myproj/myschema/fd8a2b32-170e-44e8-808f-44a8cbbc49b0/undeprecate?rev=3"
    +
    Response
    +
    +
    source{
    +  "@context": "https://bluebrain.github.io/nexus/contexts/metadata.json",
    +  "@id": "http://localhost:8080/v1/resources/myorg/myproj/fd8a2b32-170e-44e8-808f-44a8cbbc49b0",
    +  "@type": "http://localhost:8080/Custom",
    +  "_incoming": "http://localhost:8080/v1/resources/myorg/myproj/myschema/fd8a2b32-170e-44e8-808f-44a8cbbc49b0/incoming",
    +  "_outgoing": "http://localhost:8080/v1/resources/myorg/myproj/myschema/fd8a2b32-170e-44e8-808f-44a8cbbc49b0/outgoing",
    +  "_self": "http://localhost:8080/v1/resources/myorg/myproj/myschema/fd8a2b32-170e-44e8-808f-44a8cbbc49b0",
    +  "_constrainedBy": "https://bluebrain.github.io/nexus/schemas/resource",
    +  "_project": "http://localhost:8080/v1/projects/myorg/myproj",
    +  "_schemaProject": "http://localhost:8080/v1/projects/myorg/myproj",
    +  "_rev": 5,
    +  "_deprecated": false,
    +  "_createdAt": "2021-04-17T14:54:42.939Z",
    +  "_createdBy": "http://localhost:8080/v1/realms/myrealm/users/john",
    +  "_updatedAt": "2021-04-17T15:02:42.939Z",
    +  "_updatedBy": "http://localhost:8080/v1/realms/myrealm/users/john"
    +}
    +
    +

    Change schema

    +

    This operation allows to only change the schema of a resource without providing any payload.

    +
    PUT /v1/resources/{org_label}/{project_label}/{schema_id}/{resource_id}/update-schema
    +
    +

    Example

    +
    +
    Request
    +
    +
    sourcecurl -X PUT \
    +     "http://localhost:8080/v1/resources/myorg/myproj/myNewSchema/myId/update-schema"
    +
    Response
    +
    +
    source{
    +  "@context": "https://bluebrain.github.io/nexus/contexts/metadata.json",
    +  "@id": "http://localhost:8080/v1/resources/myorg/myproj/myId",
    +  "@type": "http://localhost:8080/Custom",
    +  "_incoming": "http://localhost:8080/v1/resources/myorg/myproj/myNewSchema/myId/incoming",
    +  "_outgoing": "http://localhost:8080/v1/resources/myorg/myproj/myNewSchema/myId/outgoing",
    +  "_self": "http://localhost:8080/v1/resources/myorg/myproj/myNewSchema/myId",
    +  "_constrainedBy": "http://localhost:8080/v1/resources/myorg/myproj/_/myNewSchema",
    +  "_project": "http://localhost:8080/v1/projects/myorg/myproj",
    +  "_schemaProject": "http://localhost:8080/v1/projects/myorg/myproj",
    +  "_rev": 2,
    +  "_deprecated": false,
    +  "_createdAt": "2021-04-17T14:54:42.939Z",
    +  "_createdBy": "http://localhost:8080/v1/realms/myrealm/users/john",
    +  "_updatedAt": "2021-04-17T14:56:42.939Z",
    +  "_updatedBy": "http://localhost:8080/v1/realms/myrealm/users/john"
    +}
    +

    Fetch

    GET /v1/resources/{org_label}/{project_label}/{schema_id}/{resource_id}?rev={rev}&tag={tag}
     
    @@ -698,6 +791,11 @@

    {tag}: String - the targeted tag to be fetched. This field is optional.

    {rev} and {tag} fields cannot be simultaneously present.

    +

    The {schema_id} segment allows to pass the resource schema:

    +
      +
    • If _ is provided, the value is ignored
    • +
    • If another value is provided, it must match the identifier of the resource schema.
    • +

    Example

    Request
    @@ -762,6 +860,47 @@

    Fetch remote contexts

    +

    Returns the remote contexts that have been detected during the JSON-LD resolution for this resource.

    +

    These contexts can be:

    +
      +
    • Static contexts that are statically defined in Nexus
    • +
    • Project contexts that have been registered by Nexus, in this case the entry also provides the project this context lives and its revision at the time the JSON-LD resolution has been performed
    • +
    +
    GET /v1/resources/{org_label}/{project_label}/{schema_id}/{resource_id}/remote-contexts?rev={rev}&tag={tag}
    +
    +

    where …

    +
      +
    • {rev}: Number - the targeted revision to be fetched. This field is optional and defaults to the latest revision.
    • +
    • {tag}: String - the targeted tag to be fetched. This field is optional.
    • +
    +

    {rev} and {tag} fields cannot be simultaneously present.

    +

    Example

    +
    +
    Request
    +
    +
    sourcecurl "http://localhost:8080/v1/resources/org/proj/_/my-resource/remote-contexts"
    +
    Response
    +
    +
    source{
    +  "@context": "https://bluebrain.github.io/nexus/contexts/remote-contexts.json",
    +  "remoteContexts": [
    +    {
    +      "@type": "StaticContextRef",
    +      "iri": "https://bluebrain.github.io/nexus/contexts/metadata.json"
    +    },
    +    {
    +      "@type": "ProjectContextRef",
    +      "iri": "https://localhost/nexus/context",
    +      "resource": {
    +        "id": "https://localhost/nexus/context",
    +        "project": "org/proj",
    +        "rev": 5
    +      }
    +    }
    +  ]
    +}
    +

    Fetch tags

    GET /v1/resources/{org_label}/{project_label}/{schema_id}/{resource_id}/tags?rev={rev}&tag={tag}
     
    @@ -797,6 +936,7 @@

    Within an organization

    This operation returns only resources from projects defined in the organisation {org_label} and where the caller has the resources/read permission.

    @@ -813,6 +954,7 @@

    Within all projects

    This operation returns only resources from projects where the caller has the resources/read permission.

    @@ -829,11 +972,15 @@

    Parameter description

    How to use time ranges

    A time range parameter allows to filter resources by their creation date or their last update date.

    @@ -855,6 +1002,7 @@

    {deprecated}: Boolean - can be used to filter the resulting resources based on their deprecation status
  • {rev}: Number - can be used to filter the resulting resources based on their revision value
  • {type}: Iri - can be used to filter the resulting resources based on their @type value. This parameter can appear multiple times, filtering further the @type value.
  • +
  • {typeOperator}: String (and/or) - used to determine how multiple type values affect the query, either requiring all to match (and) or any to match (or); defaults to or. See De Morgan’s laws for inference rules.
  • {createdBy}: Iri - can be used to filter the resulting resources based on their creator
  • {createdAt}: Time range - can be used to filter the resulting resources based on their creation date
  • {updatedBy}: Iri - can be used to filter the resulting resources based on the person which performed the last update
  • @@ -862,6 +1010,7 @@

    {schema}: Iri - can be used to filter the resulting resources based on the conformant schema
  • {search}: String - can be provided to select only the resources in the collection that have attribute values matching (containing) the provided string
  • {sort}: String - can be used to sort resources based on a payloads’ field. This parameter can appear multiple times to enable sorting by multiple fields. The default is done by _createdBy and @id.
  • +
  • {aggregations}: Boolean - if true then the response will only contain aggregations of the @type and _project fields; defaults to false. See Aggregations
  • Example

    @@ -896,8 +1045,83 @@

    +
    Aggregations request
    +
    +
    sourcecurl "http://localhost:8080/v1/resources/myorg?aggregations=true"
    +
    Aggregations response
    +
    +
    source{
    +  "@context": "https://bluebrain.github.io/nexus/contexts/aggregations.json",
    +  "aggregations": {
    +    "projects": {
    +      "buckets": [
    +        {
    +          "doc_count": 5,
    +          "key": "http://delta:8080/v1/projects/myorg/myproject"
    +        },
    +        {
    +          "doc_count": 5,
    +          "key": "http://delta:8080/v1/projects/myorg/myproject2"
    +        }
    +      ],
    +      "doc_count_error_upper_bound": 0,
    +      "sum_other_doc_count": 0
    +    },
    +    "types": {
    +      "buckets": [
    +        {
    +          "doc_count": 6,
    +          "key": "https://bluebrain.github.io/nexus/vocabulary/View"
    +        },
    +        {
    +          "doc_count": 2,
    +          "key": "https://bluebrain.github.io/nexus/vocabulary/CompositeView"
    +        },
    +        {
    +          "doc_count": 2,
    +          "key": "https://bluebrain.github.io/nexus/vocabulary/DiskStorage"
    +        },
    +        {
    +          "doc_count": 2,
    +          "key": "https://bluebrain.github.io/nexus/vocabulary/ElasticSearchView"
    +        },
    +        {
    +          "doc_count": 2,
    +          "key": "https://bluebrain.github.io/nexus/vocabulary/InProject"
    +        },
    +        {
    +          "doc_count": 2,
    +          "key": "https://bluebrain.github.io/nexus/vocabulary/Project"
    +        },
    +        {
    +          "doc_count": 2,
    +          "key": "https://bluebrain.github.io/nexus/vocabulary/Resolver"
    +        },
    +        {
    +          "doc_count": 2,
    +          "key": "https://bluebrain.github.io/nexus/vocabulary/SparqlView"
    +        },
    +        {
    +          "doc_count": 2,
    +          "key": "https://bluebrain.github.io/nexus/vocabulary/Storage"
    +        }
    +      ],
    +      "doc_count_error_upper_bound": 0,
    +      "sum_other_doc_count": 0
    +    }
    +  },
    +  "total": 12
     }

    +

    Aggregations

    Warning
    +

    Aggregations are experimental and the API is subject to change.

    +

    Adding the aggregations=true query parameter to a list query allows to aggregate the underlying resources by predefined terms. Currently, the following aggregations will be returned:

    +
      +
    • projects: a bucket aggregation of the resources by the project they belong to
    • +
    • types: a bucket aggregation of the @types featured in the resources
    • +
    +

    Aggregation works on the same scopes as listing (all projects, organization, and project), and only aggregates the resources for which the caller has resource/read permission.

    List filtering by schema

    This operation is only available at the project scope.

    Within a project

    @@ -907,6 +1131,7 @@

    Parameter description

    Example

    @@ -1135,7 +1361,7 @@

    @@ -1158,13 +1384,13 @@

    -
    +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    Schemas

    Schemas are rooted in the /v1/schemas/{org_label}/{project_label} collection. They define a set of rules and constraints using SHACL. Once those schemas are present, other resources can be created against them. Those resources won’t be successfully created unless they match the required constraints defined on the schema.

    @@ -946,6 +953,7 @@

    Within an organization

    This operation returns only schemas from projects defined in the organisation {org_label} and where the caller has the resources/read permission.

    @@ -958,6 +966,7 @@

    Within all projects

    This operation returns only schemas from projects defined the organisation {org_label} and where the caller has the resources/read permission.

    @@ -970,6 +979,7 @@

    Parameter description

    Example

    @@ -1061,7 +1072,7 @@

    diff --git a/docs/delta/api/search-api.html b/docs/delta/api/search-api.html index fe7d545dc7..992455deec 100644 --- a/docs/delta/api/search-api.html +++ b/docs/delta/api/search-api.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -251,7 +251,7 @@
      @@ -277,9 +277,16 @@
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    Global search

    -

    Nexus provides global search functionality across all projects through the search plugin.

    Warning
    +

    Nexus provides global search functionality across all projects through the search plugin.

    Warning

    The search plugin is experimental and its functionality and API can change without notice.

    For instructions on how to configure global search in Nexus and how it works please visit the Search configuration page.

    Query

    @@ -300,7 +307,7 @@

    -v1.8.x +Snapshot

    diff --git a/docs/delta/api/storages-api.html b/docs/delta/api/storages-api.html index 85c8322854..e69c452181 100644 --- a/docs/delta/api/storages-api.html +++ b/docs/delta/api/storages-api.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -273,7 +273,7 @@
      @@ -321,6 +321,13 @@
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    Storages

    Storages are rooted in the /v1/storages/{org_label}/{project_label} collection and are used to describe where files are physically stored.

    @@ -355,13 +362,12 @@

    Remote disk storage

    Warning

    The Remote disk storage and it remote service implementation are now deprecated and will be removed in an upcoming release.

    This storage type relies on a remote HTTP service that exposes basic file operations on an underlying POSIX file-system. This is particularly handy if your organization is running some kind of distributed network storage (such as Ceph, Gluster, GPFS, Lustre, …) that you don’t want to mount directly on the system where Nexus Delta runs.

    -

    While there’s no formal specification for this service, you can check out or deploy our own implementation: Nexus remote storage service.

    -

    In order to be able to use this storage, the configuration flag plugins.storage.storages.remote-disk.enabled should be set to true.

    +

    While there’s no formal specification for this service, you can check out or deploy our own implementation: Nexus remote storage service.

    +

    In order to be able to use this storage, the configuration flag plugins.storage.storages.remote-disk.enabled should be set to true. More information about configuration

    {
       "@type": "RemoteDiskStorage",
       "default": "{default}",
       "endpoint": "{endpoint}",
    -  "credentials": "{credentials}",
       "folder": "{folder}",
       "readPermission": "{read_permission}",
       "writePermission": "{write_permission}",
    @@ -372,7 +378,6 @@ 

  • {default}: Boolean - the flag to decide whether this storage is going to become the default storage for the target project or not.
  • {endpoint}: Uri - the endpoint where the storage service is listening to requests. This field is optional, defaulting to the configuration flag plugins.storage.storages.remote-disk.default-endpoint.
  • -
  • {credentials}: String - the service account access token to authenticate and authorize Nexus Delta client against the storage service. This field is optional, defaulting to the configuration flag plugins.storage.storages.remote-disk.default-credentials.
  • {folder}: String - the storage service bucket where files using this storage are going to be saved.
  • {read_permission}: String - the permission a client must have in order to fetch files using this storage. This field is optional, defaulting to the configuration flag plugins.storage.storages.remote-disk.default-read-permission (resources/read).
  • {write_permission}: String - the permission a client must have in order to create files using this storage. This field is optional, defaulting to the configuration flag plugins.storage.storages.remote-disk.default-write-permission (files/write).
  • @@ -385,8 +390,6 @@

  • {default}: Boolean - the flag to decide whether this storage is going to become the default storage for the target project or not.
  • {endpoint}: Uri - the Amazon S3 compatible service endpoint. This field is optional, defaulting to the configuration flag plugins.storage.storages.amazon.default-endpoint.
  • -
  • {access_key}: String - the Amazon S3 compatible access key. This field is optional, defaulting to the configuration flag plugins.storage.storages.amazon.default-access-key.
  • -
  • {secret_key}: String - the Amazon S3 compatible secret key. This field is optional, defaulting to the configuration flag plugins.storage.storages.amazon.default-secret-key.
  • {region}: String - the Amazon S3 compatible region. This field is optional, defaulting to the S3 default region configuration.
  • {read_permission}: String - the permission a client must have in order to fetch files using this storage. This field is optional, defaulting to the configuration flag plugins.storage.storages.amazon.default-read-permission (resources/read).
  • {write_permission}: String - the permission a client must have in order to create files using this storage. This field is optional, defaulting to the configuration flag plugins.storage.storages.amazon.default-write-permission (files/write).
  • @@ -771,6 +772,7 @@

    Within an organization

    This operation returns only storages from projects defined in the organisation {org_label} and where the caller has the resources/read permission.

    @@ -783,6 +785,7 @@

    Within all projects

    This operation returns only storages from projects where the caller has the resources/read permission.

    @@ -795,6 +798,7 @@

    Parameter description

    Example

    @@ -958,7 +963,7 @@

    @@ -968,7 +973,7 @@

    diff --git a/docs/delta/api/supervision-api.html b/docs/delta/api/supervision-api.html index 1138052209..d7ec7715e9 100644 --- a/docs/delta/api/supervision-api.html +++ b/docs/delta/api/supervision-api.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -250,7 +250,7 @@
      @@ -275,6 +275,13 @@
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    Supervision

    This endpoint returns information about the projections running on the current node.

    @@ -341,7 +348,7 @@

    @@ -364,13 +371,13 @@

    -
    +
    + +
    +
    +
    +
    +
    + +
    +
    +
    +
    +
    + + +
    +
    +
    +
    +
    +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +
    +
    +

    Trial

    +

    Trial operations contain read-only operations designed to help users compose and validate their resources before effectively saving them in Nexus.

    Authorization notes
    +

    When performing a request, the caller must have resources/read permission on the project each resource belongs to.

    +

    Please visit Authentication & authorization section to learn more about it.

    +

    Resource generation

    +

    This endpoint allows to create and get the output of a resource, optionally validating with an existing schema or a new one.

    +

    It applies the same validation steps than the creation/update of resources, the main difference being that nothing is persisted.

    +
    POST /v1/trial/resources/{org_label}/{project_label}
    +
    +{
    +  "schema": {schema},
    +  "resource": {resource}
    +}
    +
    +

    Where:

    +
      +
    • {schema}: String/Json: The schema to validate the provided resource. If a string is provided, it will attempt to resolve it as an existing schema. If a json payload is provided, it will attempt to generate the schema and then use the result to validate the resource. This field is optional and defaults to no SHACL validation.
    • +
    • {resource}: Json: The resource payload to test and validate
    • +
    +

    The Json response will contain:

    +
      +
    • The generated resource in the compacted JSON-LD format if the generation and the validation was successful
    • +
    • The generated schema if a new schema payload was provided
    • +
    • The error if the one of the steps fails (invalid resource/invalid new schema/existing schema not found/…)
    • +
    +

    Example

    +
    +
    Request
    +
    +
    sourcecurl -X PUT \
    +     -H "Content-Type: application/json" \
    +     "http://localhost:8080/trial/resources/myorg/myproj/" \
    +     -d \
    +'{
    +   "schema": "https://bbp.epfl.ch/nexus/schema/morphology"
    +   "resource": {
    +     "@context": [
    +       "https://neuroshapes.org",
    +       "https://bluebrain.github.io/nexus/contexts/metadata.json",
    +       {
    +         "@vocab": "https://bluebrain.github.io/nexus/vocabulary/"
    +       }
    +     ],
    +     "@id": "https://bbp.epfl.ch/nexus/data/morphology-001",
    +     "@type": "Morphology",
    +     "name": "Morphology 001"
    +   }
    + }'
    +
    Payload
    +
    +
    source{
    +  "schema": "https://bbp.epfl.ch/nexus/schema/morphology"
    +  "resource": {
    +    "@context": [
    +      "https://neuroshapes.org",
    +      "https://bluebrain.github.io/nexus/contexts/metadata.json",
    +      {
    +        "@vocab": "https://bluebrain.github.io/nexus/vocabulary/"
    +      }
    +    ],
    +    "@id": "https://bbp.epfl.ch/nexus/data/morphology-001",
    +    "@type": "Morphology",
    +    "name": "Morphology 001"
    +  }
    +}
    +
    Response
    +
    +
    source{
    +  "result": {
    +    "@context": [
    +      "https://bluebrain.github.io/nexus/contexts/metadata.json",
    +      {
    +        "@vocab": "https://bluebrain.github.io/nexus/vocabulary/"
    +      },
    +      "https://neuroshapes.org"
    +    ],
    +    "@id": "https://bbp.epfl.ch/nexus/data/morphology-001",
    +    "@type": "Morphology",
    +    "name": "Morphology 001",
    +    "_constrainedBy": "https://bbp.epfl.ch/nexus/schema/morphology",
    +    "_createdAt": "2023-09-18T12:00:00Z",
    +    "_createdBy": "http://localhost/v1/realms/wonderland/users/alice",
    +    "_deprecated": false,
    +    "_incoming": "http://localhost/v1/resources/myorg/myproj/_/https:%2F%2Fbluebrain.github.io%2Fnexus%2Fvocabulary%2FmyId/incoming",
    +    "_outgoing": "http://localhost/v1/resources/myorg/myproj/_/https:%2F%2Fbluebrain.github.io%2Fnexus%2Fvocabulary%2FmyId/outgoing",
    +    "_project": "http://localhost/v1/projects/myorg/myproj",
    +    "_rev": 1,
    +    "_schemaProject": "http://localhost/v1/projects/myorg/myproj",
    +    "_self": "http://localhost/v1/resources/myorg/myproj/_/https:%2F%2Fbluebrain.github.io%2Fnexus%2Fvocabulary%2FmyId",
    +    "_updatedAt": "2023-09-18T12:00:00Z",
    +    "_updatedBy": "http://localhost/v1/realms/wonderland/users/alice"
    +  }
    +}
    +
    +

    Validate

    +

    This operation runs validation of a resource against a schema. This would be useful to test whether resources would match the shape of a new schema.

    +
    GET /v1/resources/{org_label}/{project_label}/{schema_id}/{resource_id}/validate
    +
    +

    Example

    +
    +
    Request
    +
    +
    sourcecurl "http://localhost:8080/v1/resources/myorg/myproj/myschema/base:fd8a2b32-170e-44e8-808f-44a8cbbc49b0/validate"
    +
    Response
    +
    +
    source{
    +  "@context" : "https://bluebrain.github.io/nexus/contexts/shacl-20170720.json",
    +  "@type" : "sh:ValidationReport",
    +  "conforms" : true,
    +  "targetedNodes" : 2
    +}
    +
    +
    + + +
    +
    +
    +
    + + +
    + + + + + + + + + diff --git a/docs/delta/api/user-permissions-api.html b/docs/delta/api/user-permissions-api.html new file mode 100644 index 0000000000..17352b5860 --- /dev/null +++ b/docs/delta/api/user-permissions-api.html @@ -0,0 +1,386 @@ + + + + + + + + + + + + + + + + + + + + + + +User Permissions · Blue Brain Nexus + + + + + + + + + + + + + + + + + +
    + +
    + +
    +
    +
    +
    +
    +
    + +
      +
    • + +
    • +
    +
    +
    +
    +
    +
    +
    + + +
    +
    +
    +
    +
    +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +
    +
    +

    User Permissions

    +

    A user of delta is given certain permissions. This is done using the ACLs API

    +

    Sometimes for the sake of simplicity, it can be easier to ask whether the current user has a specific permission in a specific context. This is why the user permissions API exists

    Note
    +

    The described endpoints are experimental and the responses structure might change in the future.

    +
    +
    Requests
    +
    All requests should have no body
    +
    Responses
    +
    A response will have a 204 (no content) status code if the user is authorised
    +
    A response will have a 403 (forbidden) status code if the user is not authorised
    +
    +

    Standard permissions

    +

    This operation determines whether the current logged in user has a specific permission in a specific context

    +
    HEAD /v1/user/permissions/{org_label}/{project_label}?permission={permission}
    +
    +

    where - {permission}: String - the permission to check

    +

    Storage access permissions

    +

    This operation determines whether the current logged in user would be able to access files on a specific storage

    +
    HEAD /v1/user/permissions/{org_label}/{project_label}?storage={storage_id}&type={access_type}
    +
    +

    where - {storage_id}: String - the id of the storage - {access_type}: String - the access type of the storage. Can be read or write

    +
    + + +
    +
    +
    +
    + + +
    + + + + + + + + + diff --git a/docs/delta/api/version.html b/docs/delta/api/version.html index 62e942d3e9..b4a36643b8 100644 --- a/docs/delta/api/version.html +++ b/docs/delta/api/version.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -243,7 +243,7 @@
      @@ -252,29 +252,36 @@
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    Version

    GET /v1/version
     
    -

    This endpoint returns information about the running Delta instance: Delta version, plugin versions and service dependency versions(e.g Cassandra).

    +

    This endpoint returns information about the running Delta instance: Delta version, plugin versions and service dependency versions(e.g PostgreSQL).

    Example
    source{
       "@context": "https://bluebrain.github.io/nexus/contexts/version.json",
    -  "delta": "1.5.0",
    +  "delta": "1.9.0",
       "dependencies": {
         "blazegraph": "2.1.6-RC",
    -    "postgresql": "15.3",
    -    "elasticsearch": "8.7.1",
    -    "remoteStorage": "1.8.0"
    +    "postgresql": "15.5",
    +    "elasticsearch": "8.11.1",
    +    "remoteStorage": "1.9.0"
       },
       "plugins": {
    -    "archive": "1.8.0",
    -    "blazegraph": "1.8.0",
    -    "composite-views": "1.8.0",
    -    "elasticsearch": "1.8.0",
    -    "storage": "1.8.0"
    +    "archive": "1.9.0",
    +    "blazegraph": "1.9.0",
    +    "composite-views": "1.9.0",
    +    "elasticsearch": "1.9.0",
    +    "storage": "1.9.0"
       }
     }
    Authorization notes
    @@ -288,7 +295,7 @@

    <

    diff --git a/docs/delta/api/views/aggregated-es-view-api.html b/docs/delta/api/views/aggregated-es-view-api.html index cfb601ea50..7409d5a31b 100644 --- a/docs/delta/api/views/aggregated-es-view-api.html +++ b/docs/delta/api/views/aggregated-es-view-api.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -259,7 +259,7 @@
      @@ -293,6 +293,13 @@
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    AggregateSparqlView

    This view is an aggregate of SparqlViews. The view itself does not create any namespace, but it references the already existing namespaces of the linked SparqlViews.

    @@ -794,7 +801,7 @@

    -v1.8.x +Snapshot

    diff --git a/docs/delta/api/views/composite-sink.html b/docs/delta/api/views/composite-sink.html new file mode 100644 index 0000000000..3a4d4669bb --- /dev/null +++ b/docs/delta/api/views/composite-sink.html @@ -0,0 +1,490 @@ + + + + + + + + + + + + + + + + + + + + + + +Composite Sinks · Blue Brain Nexus + + + + + + + + + + + + + + + + + +
    + +
    + +
    +
    +
    +
    +
    +
    + +
      +
    • + +
    • +
    +
    +
    +
    + +
    +
    +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +
    +
    +

    Composite Sinks

    +

    A Composite Sink handles the following steps of composite view indexing

    +
      +
    • Querying for the graphs of resources in the Blazegraph common namespace of the view
    • +
    • Converting the obtained graphs into a format that can be pushed to a target sink
    • +
    • Finally, pushing the resources to the target sink
    • +
    +

    These steps can be implemented in different ways. In Nexus Delta, there are two kinds of Composite Sink that can be selected via configuration.

    +
      +
    1. Single Composite Sink
    2. +
    3. Batch Composite Sink
    4. +
    +

    Single Composite Sink

    +

    By default, Nexus Delta will use the Single Composite Sink. This sink performs one query to the Blazegraph common namespace for each resource in the project. The queries are done in chronological order (by the updatedAt time of the resources).

    +

    We recommend reading through the search configuration example use case and the Composite View API reference to learn more about Composite Views.

    +

    Batch Composite Sink

    +

    Starting with Delta 1.9, it is possible to configure Nexus Delta to use a Batch Composite Sink. This implementation of the Composite Sink can query the Blazegraph common namespace for multiple resource IDs at the same time.

    Note
    +

    We recommend to start using Composite Views with the default Single Composite Sink. Once you have a good understanding of it, the Batch Composite Sink can be used to enhance the performance of your deployment.

    +

    Configuring the Batch Composite Sink

    +

    In order to enable the Batch Composite Sink, configure the following Nexus Delta property:

    +

    plugins.composite-views.sink-config = batch

    +

    Furthermore, you can configure the maximum size of a batch and the maximum interval using:

    +

    plugins.composite-views.{projection-plugin}-batch.max-elements = {max-elements}

    +

    plugins.composite-views.{projection-plugin}-batch.max-interval = {max-interval}

    +

    where:

    +
      +
    • {projection-plugin} is either elasticsearch or blazegraph. The batching options of a Composite Sink can be set separately for each target type.
    • +
    • {max-elements} is the maximum number of elements to batch at once (defaults to 10)
    • +
    • {max-interval} is the maximum interval of time to wait for {max-elements} elements
    • +
    +

    How to write a SPARQL construct query for the Batch Composite Sink

    +

    In order to use the Batch Composite Sink successfully, it is necessary to rework some aspects of a regular SPARQL construct query. We explain the changes through an example.

    +

    Example

    +

    Suppose we are in a situation where Composite Views are using the Single Composite Sink and have the following query:

    +
    PREFIX schema: <http://schema.org/>
    +PREFIX nxv: <https://bluebrain.github.io/nexus/vocabulary/>
    +CONSTRUCT {
    +  ?id     nxv:name   ?name   ;
    +          nxv:age    ?age    .
    +          nxv:parent ?parent .
    +} WHERE {
    +  BIND({resource_id} AS ?id) .
    +  ?id schema:name ?name .
    +  
    +  OPTIONAL { ?id schema:age ?age }
    +  OPTIONAL { ?id schema:parent ?parent . }
    +}
    +
    +

    Using the default Single Composite Sink, Nexus Delta will query the resources Alice and Bob individually and obtain the following n-triples from Blazegraph:

    +
    <http://people.com/Alice> <http://schema.org/name> <Alice>
    +<http://people.com/Alice> <http://schema.org/parent> <http://people.com/Bob>
    +
    +
    <http://people.com/Bob> <http://schema.org/name> <Bob>
    +<http://people.com/Bob> <http://schema.org/age> <42>
    +
    +

    In particular, note that when looking at the graph of Alice, we do not know the age of Bob.

    +

    The first change to introduce in order to make this query work with batches of resources is to replace the BIND({resource_id} AS ?id) . with VALUES ?id { {resources_id} }. Nexus Delta will use this template to replace {resource_id} with multiple resource in case it receives a batch of more than one element. The query is now:

    +
    PREFIX schema: <http://schema.org/>
    +PREFIX nxv: <https://bluebrain.github.io/nexus/vocabulary/>
    +CONSTRUCT {
    +  ?id     nxv:name   ?name   ;
    +          nxv:age    ?age    .
    +          nxv:parent ?parent .
    +} WHERE {
    +  VALUES ?id { {resource_id} } .
    +  ?id schema:name ?name .
    +  
    +  OPTIONAL { ?id schema:age ?age }
    +  OPTIONAL { ?id schema:parent ?parent . }
    +}
    +
    +

    With the Batch Composite Sink enabled, if Alice and Bob are batched together, this query will result in the following triples:

    +
    <http://people.com/Alice> <http://schema.org/name> <Alice>
    +<http://people.com/Alice> <http://schema.org/parent> <http://people.com/Bob>
    +<http://people.com/Bob> <http://schema.org/name> <Bob>
    +<http://people.com/Bob> <http://schema.org/age> <42>
    +
    +

    Note how the results are the merged result of the individual queries. While we were able to query several resources simultaneously, we are now facing a framing problem. If we try to frame http://people.com/Alice, its graph now contains more information than before; it will now include the age of Bob, something that we did not request.

    +

    In order to solve this problem, we will introduce aliasing for the root resource IDs. The query will now become:

    +
    PREFIX schema: <http://schema.org/>
    +PREFIX nxv: <https://bluebrain.github.io/nexus/vocabulary/>
    +CONSTRUCT {
    +  ?alias  nxv:name   ?name   ;
    +          nxv:age    ?age    .
    +          nxv:parent ?parent .
    +} WHERE {
    +  VALUES ?id { {resource_id} } .
    +  BIND(IRI(CONCAT(STR(?id), '/alias')) AS ?alias) .
    +  
    +  ?id schema:name ?name .
    +  
    +  OPTIONAL { ?id schema:age ?age }
    +  OPTIONAL { ?id schema:parent ?parent }
    +}
    +
    +

    With this query, a batch query for both Alice and Bob will now yield:

    +
    <http://people.com/Alice/alias> <http://schema.org/name> <Alice>
    +<http://people.com/Alice/alias> <http://schema.org/parent> <http://people.com/Bob>
    +<http://people.com/Bob/alias> <http://schema.org/name> <Bob>
    +<http://people.com/Bob/alias> <http://schema.org/age> <42>
    +
    +

    You can see that the root node of Bob’s graph is now http://people.com/Bob/alias, while Alice’s parent is http://people.com/Bob. This distinction ensures that we cannot get Bob’s age by looking at Alice’s graph, thus reproducing the behavior that we had with the Single Composite Sink.

    +

    Nexus Delta takes care of framing these results so that the framed documents will be the same as with the Single Composite Sink, and will not contain any alias keyword. For example, for the resource http://people.com/Alice, Nexus Delta will obtain its graph by looking at the http://people.com/Alice/alias root node, and use the resulting graph (removing the /alias part) for JSON-LD framing.

    +

    Summary

    +

    To use the Batch Composite Sink the following changes are necessary:

    +
      +
    • BIND({resource_id} AS ?id) must become VALUES ?id { {resource_id} } to allow for batches of resources to be queried from Blazegraph.
    • +
    • BIND(IRI(CONCAT(STR(?id), '/alias')) AS ?alias) needs to be added, and the relevant (?id) “root nodes” replaced by ?alias in the CONSTRUCT part of the query. This is done in order to avoid any clashes between the graphs of several resources. Do note that in case you have several “root nodes” in the CONSTRUCT part of your construct query, you might need several aliases.
    • +
    +
    + + +
    +
    +
    +
    + + +
    + + + + + + + + + diff --git a/docs/delta/api/views/composite-view-api.html b/docs/delta/api/views/composite-view-api.html index 87054f7da0..a77f893ed5 100644 --- a/docs/delta/api/views/composite-view-api.html +++ b/docs/delta/api/views/composite-view-api.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
  • +
  • Batching queries to the intermediate space
  • Payload
  • +
  • Batching queries to the intermediate space
  • Payload
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    CompositeView

    This view is composed by multiple sources and projections.

    @@ -466,7 +475,6 @@

    Sparql Construct Query - Defines the Sparql query to execute against the intermediate Sparql space for each target resource.
  • {resourceSchema}: Iri - Selects only resources that are validated against the provided schema Iri to perform the query. This field is optional.
  • {resourceType}: Iri - Select only resources of the provided type Iri to perform the query. This field is optional.
  • -
  • {tag}: String - Selects only resources with the provided tag to perform the query. This field is optional.
  • {includeMetadata}: Boolean - If true, the resource’s nexus metadata (_constrainedBy, _deprecated, …) will be included in the ElasticSearch document. The default value is false.
  • {includeDeprecated}: Boolean - If true, deprecated resources are also indexed. The default value is false.
  • {permission}: String - the permission necessary to query this projection. Defaults to views/query.
  • @@ -507,7 +514,6 @@

    {projectionId}: Iri - The identifier of the projection. This field is optional. When missing, a randomly generated Iri will be assigned.
  • {resourceSchema}: Iri - Selects only resources that are validated against the provided schema Iri to perform the query. This field is optional.
  • {resourceType}: Iri - Select only resources of the provided type Iri to perform the query. This field is optional.
  • -
  • {tag}: String - Selects only resources with the provided tag to perform the query. This field is optional.
  • {includeMetadata}: Boolean - If true, the resource’s nexus metadata (_constrainedBy, _deprecated, …) will be stored in the ElasticSearch document. Otherwise it won’t. The default value is false.
  • {includeDeprecated}: Boolean - If true, deprecated resources are also indexed. The default value is false.
  • {query}: Sparql Construct Query - Defines the Sparql query to execute against the intermediate Sparql space for each target resource.
  • {permission}: String - the permission necessary to query this projection. Defaults to views/query.
  • +

    Batching queries to the intermediate space

    +

    The queries that projections perform to the intermediate Sparql space can be either executed per individual resource, or in batches containing multiple resources. To learn more about it, please refer to the Composite Sink page.

    Payload

    {
       "@id": "{someid}",
    @@ -2448,7 +2455,7 @@ 

    @@ -2471,13 +2478,13 @@

    -
    +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    ElasticSearchView

    This view creates an ElasticSearch index and stores the targeted Json resources into an ElasticSearch Document.

    @@ -1092,6 +1101,37 @@

    +

    Fetch Elasticsearch mapping

    +
    GET /v1/views/{org_label}/{project_label}/{view_id}/_mapping
    +
    +

    Retrieves the view’s ElasticSearch mapping.

    +

    Example

    +
    +
    Request
    +
    +
    sourcecurl "http://localhost:8080/v1/views/myorg/myproj/myview/_mapping"
    +
    Response
    +
    +
    source{
    +  "delta_f488d131-5f29-4721-979e-277a0850fed2_1": {
    +    "mappings": {
    +      "properties": {
    +        "title": {
    +          "type": "text",
    +          "fields": {
    +            "keyword": {
    +              "type": "keyword"
    +            }
    +          }
    +        },
    +        "description": {
    +          "type": "text"
    +        }
    +      }
    +    }
    +  }
    +}
    +

    Fetch tags

    GET /v1/views/{org_label}/{project_label}/{view_id}/tags?rev={rev}&tag={tag}
     
    @@ -1208,7 +1248,7 @@

    diff --git a/docs/delta/api/views/index.html b/docs/delta/api/views/index.html index 46a7853484..a70d56bf01 100644 --- a/docs/delta/api/views/index.html +++ b/docs/delta/api/views/index.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
  • @@ -267,7 +273,7 @@
    @@ -288,18 +294,24 @@
  • AggregateSparqlView
  • CompositeView
  • Indexing
  • -
  • Passivation
  • List views
  • -
  • Server Sent Events +
  • Indexing failures
  • +
  • Server Sent Events
  • @@ -309,6 +321,13 @@
    +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    Views

    Views are rooted in the /v1/views/{org_label}/{project_label} collection and are used to index the selected resources into a bucket.

    @@ -343,10 +362,6 @@

    async - (default value) the view will be indexed asynchronously
  • sync - the view will be indexed synchronously and the API call won’t return until the indexing is finished
  • -

    Passivation

    -

    Views are now stopped if no event has been detected for a given period (by default 30 minutes).

    -

    This value can be updated by updating the configuration keys: - plugins.blazegraph.idle-timeout - plugins.composite-views.idle-timeout - plugins.elasticsearch.idle-timeout

    -

    Setting an infinite time (Inf) will disable this feature.

    List views

    There are three available endpoints to list views in different scopes.

    Within a project

    @@ -359,6 +374,7 @@

    Within an organization

    This operation returns only views from projects defined in the organisation {org_label} and where the caller has the resources/read permission.

    @@ -371,6 +387,7 @@

    Within all projects

    This operation returns only views from projects where the caller has the resources/read permission.

    @@ -383,8 +400,9 @@

    -

    Parameter description

    +

    Parameter description

    Example

    @@ -469,8 +488,76 @@

    +

    Indexing failures

    +

    Listing indexing failures

    +

    This endpoint returns the indexing failures for the view {view_id}. The caller must have view/write permission on {org_label}/{project_label}.

    +
    GET /v1/views/{org_label}/{project_label}/{view_id}/failures
    +    ?from={from}
    +    &size={size}
    +    &instant={instant}
    +
    +

    Parameter description

    +
      +
    • {from}: Number - is the parameter that describes the offset for the current query; defaults to 0
    • +
    • {size}: Number - is the parameter that limits the number of results; defaults to 20
    • +
    • {instant}: Time Range - can be used to filter the results based on the moment they were risen
    • +
    +

    Example

    +
    +
    Request
    +
    +
    sourcecurl "http://localhost:8080/v1/views/myorg/myproj/myview/failures"
    +
    Response
    +
    +
    source{
    +  "@context": [
    +    "https://bluebrain.github.io/nexus/contexts/metadata.json",
    +    "https://bluebrain.github.io/nexus/contexts/search.json",
    +    "https://bluebrain.github.io/nexus/contexts/error.json"
    +  ],
    +  "_total": 2,
    +  "_results": [
    +    {
    +      "errorType": "java.lang.Exception",
    +      "id": "https://bluebrain.github.io/nexus/vocabulary/myid",
    +      "message": "The resource could not be indexed for the following reason: '...'",
    +      "offset": {
    +        "@type": "At",
    +        "value": 42
    +      },
    +      "project": "myorg/myproj",
    +      "_rev": 1
    +    },
    +    {
    +      "errorType": "java.lang.Exception",
    +      "id": "https://bluebrain.github.io/nexus/vocabulary/myid",
    +      "message": "The resource could not be indexed for the following reason: '...'",
    +      "offset": {
    +        "@type": "At",
    +        "value": 42
    +      },
    +      "_rev": 1
    +    }
    +  ]
    +}
    +
    +

    Fetch indexing failures as SSEs

    +

    This endpoint fetches the available indexing failures. The Last-Event-Id is optional and provides the id of the indexing failure at which to start the stream; by default all indexing failures are fetched. The caller must have view/write permission on {org_label}/{project_label}.

    +
    GET /v1/views/{org_label}/{project_label}/{view_id}/failures/sse
    +
    +

    Example

    +
    +
    Request
    +
    +
    sourcecurl "http://localhost:8080/v1/views/myorg/myproj/myview/failures/sse"
    +
    Response
    +
    +
    sourcedata:{"@context":"https://bluebrain.github.io/nexus/contexts/error.json","errorType":"java.lang.Exception","id":"myid","message":"boom","offset":{"@type":"At","value":42}}
    +event:IndexingFailure
    +id:1
    +

    Server Sent Events

    -

    From Delta 1.5, it is possible to fetch SSEs for all resolvers or just resolvers in the scope of an organization or a project.

    +

    From Delta 1.5, it is possible to fetch SSEs for all views or just views in the scope of an organization or a project.

    GET /v1/views/events # for all view events in the application
     GET /v1/views/{org_label}/events # for view events in the given organization
     GET /v1/views/{org_label}/{project_label}/events # for view events in the given project
    @@ -505,21 +592,6 @@ 

    -

    Fetch indexing failures

    -

    This endpoint fetches the available indexing failures as SSEs. The Last-Event-Id is optional and provides the id of the indexing failure at which to start the stream; by default all indexing failures are fetched. The caller must have view/write permission on {org_label}/{project_label}.

    -
    GET /v1/views/{org_label}/{project_label}/{view_id}/failures
    -
    -

    Example

    -
    -
    Request
    -
    -
    sourcecurl "http://localhost:8080/v1/views/myorg/myproj/myview/failures"
    -
    Response
    -
    -
    sourcedata:{"@context":"https://bluebrain.github.io/nexus/contexts/error.json","errorType":"java.lang.Exception","id":"myid","message":"boom","offset":{"@type":"At","value":42}}
    -event:IndexingFailure
    -id:1
    -
    diff --git a/docs/delta/api/views/pipes.html b/docs/delta/api/views/pipes.html index 6ed3ca0fa7..bc06939097 100644 --- a/docs/delta/api/views/pipes.html +++ b/docs/delta/api/views/pipes.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -261,7 +261,7 @@
      @@ -297,6 +297,13 @@
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    Elasticsearch Pipes

    Pipes are the processing units of a pipeline for an Elasticsearch view.

    @@ -400,8 +407,8 @@

    Elem source for documentation about this class. -
  • GraphResource source for documentation about this class.
  • +
  • Elem source for documentation about this class.
  • +
  • GraphResource source for documentation about this class.
  • Please visit Plugins to learn about how to create/package/deploy a plugin.

    Inside this plugin, you can then define additional pipes:

    @@ -516,7 +523,7 @@

    here and the associated unit tests here.

    +

    The source code for the core pipes is available here and the associated unit tests here.

    diff --git a/docs/delta/api/views/sparql-view-api.html b/docs/delta/api/views/sparql-view-api.html index 5bd7abd548..4ee3e8ee47 100644 --- a/docs/delta/api/views/sparql-view-api.html +++ b/docs/delta/api/views/sparql-view-api.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -266,7 +266,7 @@
      @@ -307,6 +307,13 @@
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    Architecture

    Blue Brain Nexus is a collection of software components that address various organizational needs relating to data storage, management, analysis and consumption. It was designed to support the data-driven science iterative cycle at Blue Brain but its genericity allows for its use in arbitrary contexts.

    @@ -305,7 +312,7 @@

    cluster configuration by updating the number of nodes and defining the index for each of them +
  • Change the cluster configuration by updating the number of nodes and defining the index for each of them
  • Start the cluster
  • Clustering

    New clustering deployment
    @@ -356,7 +363,7 @@

    <

    Resource identification is based on HTTP Internationalized Resource Identifiers (IRIs) and uniqueness is guaranteed within the scope of a project. This allows the system to be used in a multi-tenant configuration but at the same time it implies that project and organization identifiers are part of a resource addressing scheme.

    In order to avoid limitations in URL lengths and for convenience, resource identifiers can be aliased and compacted (CURIE) using project level configurations.

    Authentication and Authorization

    -

    The system supports OpenID Connect, OAuth 2.0 and JSON Web Tokens (JWTs) standards and can be configured to use identity providers that support these standards. Proof of identity can be provided by passing a Bearer JWT in the Authorization header of the HTTP requests when consuming the RESTful API.

    +

    The system supports OpenID Connect, OAuth 2.0 and JSON Web Tokens (JWTs) standards and can be configured to use identity providers that support these standards. Proof of identity can be provided by passing a Bearer JWT in the Authorization header of the HTTP requests when consuming the RESTful API.

    Nexus Delta can use LDAP as an identity management system through several off-the-shelf products that implement these protocols on top of LDAP, like for example Keycloak.

    The authorization flow is as follows:

    diff --git a/docs/delta/benchmarks/index.html b/docs/delta/benchmarks/index.html index b10b00b866..6a9c655963 100644 --- a/docs/delta/benchmarks/index.html +++ b/docs/delta/benchmarks/index.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -243,7 +243,7 @@
      @@ -252,6 +252,13 @@
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    Benchmarks

    The main goal of the benchmarks is to analyze the hardware requirements for a Nexus deployment and to find potential issues and / or bottlenecks. In particular, we are most interested in the following metrics:

    @@ -275,7 +282,7 @@

    -v1.8.x +Snapshot

    diff --git a/docs/delta/benchmarks/v1.2.1.html b/docs/delta/benchmarks/v1.2.1.html index c2302a08ac..a828b8a525 100644 --- a/docs/delta/benchmarks/v1.2.1.html +++ b/docs/delta/benchmarks/v1.2.1.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -265,7 +265,7 @@
      @@ -305,6 +305,13 @@
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    Nexus Delta

    Blue Brain Nexus Delta is a low latency, scalable and secure system that realizes a range of functions to support data management and knowledge graph lifecycles.

    It is a central piece of the Nexus ecosystem of software components as it offers a set of foundational capabilities to the other components (Nexus Fusion and Nexus Forge) around data and metadata storage, management, validation and consumption in a secure setting.

    -

    Nexus Delta is developed in the open with a permissive licence (Apache License, version 2.0) using open standards and interoperable semantic web technologies like OpenID Connect, OAuth 2.0, RDF, JSON-LD, SHACL, Server-Sent Events.

    +

    Nexus Delta is developed in the open with a permissive licence (Apache License, version 2.0) using open standards and interoperable semantic web technologies like OpenID Connect, OAuth 2.0, RDF, JSON-LD, SHACL, Server-Sent Events.

    It is quite versatile as it is able to handle very small to very large amounts of data on-premise or in the cloud and can be used in a large spectrum of industries being completely domain agnostic.

    Please refer to the architecture and api reference sections for more information about this component.

    @@ -267,7 +274,7 @@

    -v1.8.x +Snapshot

    diff --git a/docs/delta/metadata.html b/docs/delta/metadata.html new file mode 100644 index 0000000000..d54fe3d91d --- /dev/null +++ b/docs/delta/metadata.html @@ -0,0 +1,387 @@ + + + + + + + + + + + + + + + + + + + + + + +Nexus Metadata · Blue Brain Nexus + + + + + + + + + + + + + + + + + +
    + +
    + +
    +
    +
    + +
    +
    +
    + + +
    +
    +
    +
    +
    +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +
    +
    +

    Nexus Metadata

    +

    Nexus Delta entities fall in two categories:

    + +

    Upon creation of these entities, Nexus Delta will create metadata fields which are described below.

    +
      +
    • _self: unique address of the entity across Nexus Delta +
        +
      • Nexus Delta follows the HATEOAS architecture, which is reflected in the _self address being discoverable in Nexus Delta’s different responses. Example: from a resources listing operation, the _self endpoints listed can be used to fetch the underlying resources
      • +
      +
    • +
    +

    Auditing

    +

    The following metadata can help to audit an entity.

    +
      +
    • _rev: the revision number of the entity
    • +
    • _deprecated: boolean indicating whether or not the entity is deprecated
    • +
    • _createdAt: datetime at which the entity was first created
    • +
    • _createdBy: the user that first created the entity
    • +
    • _updatedAt: datetime at which the entity was last updated
    • +
    • _updatedBy: the user that last updated the entity
    • +
    +
    + + +
    +
    +
    +
    + + +
    + + + + + + + + + diff --git a/docs/delta/plugins/index.html b/docs/delta/plugins/index.html index 235132cc4b..17330cecfd 100644 --- a/docs/delta/plugins/index.html +++ b/docs/delta/plugins/index.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -259,7 +259,7 @@
      @@ -293,17 +293,24 @@
      +
      +
      Snapshot version
      +

      +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

      +

      Plugins

      Starting from version 1.5, Delta introduces the ability to extend its functionality using plugins. Plugins enable developers to add new functionality to Nexus Delta without the need to modify Delta itself. Plugins can introduce various new functionalities, including new API endpoints and indexing capabilities.

      Note

      Plugins are still an experimental feature and Delta SDKs and dependent modules(rdf, sourcing) provide no binary compatibility guarantees.

      Plugin development

      Plugins used by Delta need to be packaged as a .jar file containing the plugin code with all its dependencies. Delta loads plugins from .jar files located in a directory specified by DELTA_PLUGINS environment variable.

      -

      Plugins must define exactly one class which extends PluginDef trait.

      +

      Plugins must define exactly one class which extends PluginDef trait.

      The class must define following methods:

      def info: PluginDescription
       
      -

      this method returns instance of PluginDescription which defines the plugin name and version.

      +

      this method returns instance of PluginDescription which defines the plugin name and version.

      def initialize(locator: Locator): Task[Plugin]
       

      this method can be used to initialize the plugin and returns an instance of a Plugin, which can additionally define logic to stop the plugin gracefully.

      @@ -338,13 +345,13 @@

      sourcepackage ch.epfl.bluebrain.nexus.delta.testplugin
       
      +import cats.effect.IO
       import ch.epfl.bluebrain.nexus.delta.sdk.PriorityRoute
       import ch.epfl.bluebrain.nexus.delta.sdk.model.ComponentDescription.PluginDescription
       import ch.epfl.bluebrain.nexus.delta.sdk.model.{BaseUri, Name}
       import ch.epfl.bluebrain.nexus.delta.sdk.plugin.{Plugin, PluginDef}
       import izumi.distage.model.Locator
       import izumi.distage.model.definition.ModuleDef
      -import monix.bio.Task
       
       case class TestPluginDef() extends PluginDef {
       
      @@ -361,7 +368,7 @@ 

      @@ -374,21 +381,21 @@

      libraryDependencies += "ch.epfl.bluebrain.nexus" %% "delta-sdk" % deltaVersion % Provided

      Dependency injection

      -

      Delta uses distage library for dependency injection. Each plugin must define ModuleDef to create instances of its own classes. All the dependencies provided by ModuleDefs defined in Delta modules, as well as other plugins can be used here.

      +

      Delta uses distage library for dependency injection. Each plugin must define ModuleDef to create instances of its own classes. All the dependencies provided by ModuleDefs defined in Delta modules, as well as other plugins can be used here.

      The plugin can also define instances of following traits/classes, which will be used in Delta:

        -
      • PriorityRoute - allows the plugin to define Akka HTTP Route with priority. The priority is used by Delta to prioritize route evaluation
      • -
      • ScopeInitialization - allows the plugin to define hooks which will be run on organization and project creation.
      • -
      • ScopedEntityDefinition - allows to define the required information to be able to handle a custom scoped entity
      • -
      • Serializer - allows to define how to serialize and deserialize an event / a state to database
      • -
      • ResourceShift - enables Delta to retrieve the different resources in a common format for tasks like indexing or resolving operations.
      • -
      • SseEncoder - enables Delta to convert a database event to a SSE event
      • -
      • EventMetricEncoder - enables Delta to convert a database event to an event metric
      • -
      • MetadataContextValue - registers metadata context of this plugin into global metadata context
      • -
      • RemoteContextResolution - enables Delta to resolve static contexts defined by the plugin
      • -
      • ServiceDependency - allows the plugin to define dependencies which will be displayed in /version endpoint.
      • -
      • ApiMappings - allows the plugin to define default API mappings used to shorten URLs
      • -
      • ResourceToSchemaMappings - allows the plugin to define mapping from the resource type to schema, which can be used to interact with resources created by the plugin through /resources endpoints.
      • +
      • PriorityRoute - allows the plugin to define Akka HTTP Route with priority. The priority is used by Delta to prioritize route evaluation
      • +
      • ScopeInitialization - allows the plugin to define hooks which will be run on organization and project creation.
      • +
      • ScopedEntityDefinition - allows to define the required information to be able to handle a custom scoped entity
      • +
      • Serializer - allows to define how to serialize and deserialize an event / a state to database
      • +
      • ResourceShift - enables Delta to retrieve the different resources in a common format for tasks like indexing or resolving operations.
      • +
      • SseEncoder - enables Delta to convert a database event to a SSE event
      • +
      • EventMetricEncoder - enables Delta to convert a database event to an event metric
      • +
      • MetadataContextValue - registers metadata context of this plugin into global metadata context
      • +
      • RemoteContextResolution - enables Delta to resolve static contexts defined by the plugin
      • +
      • ServiceDependency - allows the plugin to define dependencies which will be displayed in /version endpoint.
      • +
      • ApiMappings - allows the plugin to define default API mappings used to shorten URLs
      • +
      • ResourceToSchemaMappings - allows the plugin to define mapping from the resource type to schema, which can be used to interact with resources created by the plugin through /resources endpoints.

      Class loading

      In order to avoid clashes between different plugins, Delta uses custom classloader to load plugin classes, which will load classes from the plugin first, then using application classloader and other plugins after that. It is therefore recommended to not include in the plugin jar any dependencies which are also provided by SDK. Libraries can be easily excluded from dependencies in sbt:

      @@ -406,14 +413,14 @@

      Existing plugins

      Currently, following Delta functionality is provided by plugins:

      Elasticsearch plugin is required in order to provide listings in the API, other plugins can be excluded if their functionality is not needed. All the above plugins are included in the Delta Docker image.

      @@ -424,7 +431,7 @@

      @@ -434,7 +441,7 @@

      diff --git a/docs/faq.html b/docs/faq.html index c7e8b28a71..cd0f4a27d7 100644 --- a/docs/faq.html +++ b/docs/faq.html @@ -154,6 +154,8 @@

  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -275,7 +275,7 @@
      @@ -325,6 +325,13 @@
      +
      +
      Snapshot version
      +

      +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

      +

      FAQ

      General FAQ

      @@ -391,7 +398,7 @@

      Can I connect any SPARQL client to Nexus’ SPARQL endpoint?

      Yes. As long as the client supports the ability to provide a Authentication HTTP Header (for authentication purposes) on the HTTP request, any SPARQL client should work.

      How can I create an organization as an anonymous user in the docker-compose file? What needs to be done to switch to “authenticated” mode?

      -

      By default, the permissions used - for an authenticated user - when running Nexus Delta are the ones defined on the JVM property app.permissions.minimum. In order to change that behaviour, please create some ACLs for the path /. For more details about ACLs creation, visit the ACLs page.

      +

      By default, the permissions used - for an authenticated user - when running Nexus Delta are the ones defined on the JVM property app.permissions.minimum. In order to change that behaviour, please create some ACLs for the path /. For more details about ACLs creation, visit the ACLs page.

      Can I use Blue Brain Nexus from Jupyter Notebooks?

      Blue Brain Nexus can be used from Jupyter Notebooks using Nexus Forge or Nexus Python SDK. Alternatively, you can also use any Python HTTP client and use Nexus REST API directly from the Jupyter Notebook. Please consider looking at our tutorial to learn how to user Nexus Forge on the Sandbox. Other examples are provided in the folder Notebooks.

      @@ -402,7 +409,7 @@

      -v1.8.x +Snapshot

      diff --git a/docs/forge.html b/docs/forge.html index a65ce99689..9f722e6b35 100644 --- a/docs/forge.html +++ b/docs/forge.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
  • @@ -253,7 +261,7 @@
    @@ -271,7 +279,15 @@
  • Technology Overview
  • Pages
  • Plugins
  • -
  • Customization
  • +
  • Customization +
  • @@ -281,36 +297,52 @@
    +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    Architecture

    Technology Overview

    -

    Nexus Fusion is a server-side rendered single-page webapp powered by React.js. It is written in Typescript, and therefore must be transpiled into native browser JavaScript, during a build step, before being served.

    -

    The build step produces a server artifact to run on a Node.js server, with client-side assets and Javascript.

    -

    We produce a docker image that is able to serve the compiled assets directly, which is available on Dockerhub.

    +

    Nexus Fusion is a partially server-side rendered single-page web app powered by React.js. It is written in TypeScript, and therefore must be transpiled into native browser JavaScript, during a build step, before being served.

    +

    The build step produces a server artifact to run on a Node.js server, with client-side assets and JavaScript.

    +

    We produce a Docker image that is able to serve the compiled assets directly, which is available on Dockerhub.

    Although the application is served by a Node.js server, the client communicates directly to Nexus Delta using Nexus.js.

    Pages

    -

    Nexus Fusion has undergone a significant restructuring to transition from a SubApp-based architecture to a page-based structure. This change will prepare the application for a full migration to file-system based routing technologie and help seperating the application main features.

    -

    Pages serve a specific top level entity or functionality concerns of diverse users and activities, and provide varying access privileges to each. The main features that can be accessed from the home page are: Organizations, Projects, Studios and My data.

    -

    The page Project, is responsible for handling all aspects of managing a single project.

    -

    The search page has been removed, but every element that was previously available on the search page now has a link on the home page. These links redirect users to the appropriate global search type page.

    -

    The pages in Nexus Fusion are part of the source code and reside in the src/pages folder.

    -

    While all the previous feature is still available in the current version of Nexus Fusion, we expect its functionality to evolve and change in the upcoming release.

    +

    Nexus Fusion has undergone a significant restructuring to transition from a SubApp-based architecture to a page-based structure. This change will prepare the application for a full migration to file-system based routing system and help separate the application’s main features.

    +

    Pages serve a specific top level entity or functionality concerns of diverse users and activities, and provide varying access privileges to each. The main features that can be accessed from the home page are: Organizations, Projects, Studios and My data.

    +

    The page Project, is responsible for handling all aspects of managing a single project.

    Change of search page
    +

    The search page has been removed, but every element that was previously available on the search page now has a link on the home page. These links redirect users to the appropriate global search type page.

    +

    The pages in Nexus Fusion are part of the source code and reside in the src/pages folder. While all the previous features are still available in the current version of Nexus Fusion, we expect their functionality to evolve and change in the upcoming releases.

    Plugins

    Plugins are ways to render resources. You can find more about them here. It is important to note that the plugin repository is hosted separately from Nexus Fusion. Nexus Fusion will request a Plugin Manifest from this repository at run-time, and fetch plugins to render during run time based on a config. Both the plugins, the configuration, and the manifest should be hosted somewhere Nexus Fusion can request it.

    Customization

    -

    You can customize the Header of Nexus Fusion by setting up the following environment variables:

    +

    Customize the Nexus codebase appearance by setting environment variables. Defaults are provided by Nexus Fusion if not set.

    +

    Environment Variables for Customization

    +

    Nexus Header

    +

    Landing Page Customization

    +
      +
    • LOGO_IMG: HTTPS URL for the application logo. Recommended: SVG format with transparent background. Ideal size: ~35 px height, max 250 px width. Click here for an example SVG.
    • +
    • LOGO_LINK: HTTPS URL redirecting from the logo. Example: https://www.epfl.ch
    • +
    • LANDING_VIDEO: HTTPS URL for a landing page video. Requirements: MP4 format, H.264 codec, ~10MB, 1920×1080 resolution. Click here for an example video. Ensure that the video’s main color is not too bright, as the text on top of it will be white.
    • +
    • LANDING_POSTER_IMG: HTTPS URL for a loading image on the landing page, displayed while the video loads. Click here for an example poster image. Please use a PNG or JPG image that matches the size and color of the video. Aim for the image to have the same dimensions as the video. Ensure that the image size is kept under 200 KB to improve loading time and user experience.
    • +
    • MAIN_COLOR: Main background color in hex code. Example color: #062d68
    • +
    +

    Page Specific Customization

    +

    Example organization env Example projects env Example studios env

    +
      +
    • ORGANIZATION_IMG: Provide the HTTPS URL for an image on the organization page. Click here for an example image. The recommended dimensions are approximately 1500 × 450 pixels. Please ensure that the image size is kept under 200 KB to improve loading time and user experience.
    • +
    • PROJECTS_IMG: Share the HTTPS URL for an image on the projects page. Click here for an example image. The recommended dimensions are approximately 1500 × 450 pixels. Please ensure that the image size is kept under 200 KB to improve loading time and user experience.
    • +
    • STUDIOS_IMG: Provide the HTTPS URL for an image on the studios page. Click here for an example image. The recommended dimensions are approximately 1500 × 450 pixels. Please ensure that the image size is kept under 200 KB to improve loading time and user experience.
    • +
    +

    Additional Options

      -
    • LOGO_IMG: Url for an image to be used as application logo in the Header, for example, https://www.epfl.ch/logo-img.png
    • -
    • LOGO_LINK: Url for the logo, for example, https://www.epfl.ch
    • -
    • ORGANIZATION_IMG: Url for the organization page foreground image, for example, https://www.epfl.ch/default-org-img.png
    • -
    • PROJECTS_IMG: Url for the projects page foreground image, for example,https://www.epfl.ch/default-projects-img.png
    • -
    • STUDIOS_IMG: Url for the studios page foreground image, for example, https://www.epfl.ch/default-studios-img.png
    • -
    • LANDING_VIDEO: Url for video in the the landing page, for example, https://www.epfl.ch/landing-page-video.mp4
    • -
    • LANDING_POSTER_IMG: Url for the video’s poster image in landing page (replace the video when loading, for example,https://www.epfl.ch/landing-page-poster-img.png
    • -
    • MAIN_COLOR: Url for the organization page, for example “#062d68”
    • -
    • If you use Nexus Forge, it is possible to include a Forge templates button by providing the url as FORGE_LINK, for example, https://some-url.hi
    • +
    • FORGE_LINK: HTTPS URL for the Forge templates button (only if using Nexus Forge). Example: https://some-url.com
    -

    The full list of environment variables can be found here.

    +

    For a comprehensive list of environment variables, see the Environment Variables List.

    diff --git a/docs/fusion/assets/environment-variables.png b/docs/fusion/assets/environment-variables.png new file mode 100644 index 0000000000..e12b52d2df Binary files /dev/null and b/docs/fusion/assets/environment-variables.png differ diff --git a/docs/fusion/assets/organizations-envs.png b/docs/fusion/assets/organizations-envs.png new file mode 100644 index 0000000000..56efa2e09b Binary files /dev/null and b/docs/fusion/assets/organizations-envs.png differ diff --git a/docs/fusion/assets/projects-envs.png b/docs/fusion/assets/projects-envs.png new file mode 100644 index 0000000000..ee968a485c Binary files /dev/null and b/docs/fusion/assets/projects-envs.png differ diff --git a/docs/fusion/assets/studios-envs.png b/docs/fusion/assets/studios-envs.png new file mode 100644 index 0000000000..0af24e64ff Binary files /dev/null and b/docs/fusion/assets/studios-envs.png differ diff --git a/docs/fusion/index.html b/docs/fusion/index.html index 07ae2510f6..ba774a0532 100644 --- a/docs/fusion/index.html +++ b/docs/fusion/index.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -251,7 +251,7 @@
      @@ -277,6 +277,13 @@
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    Nexus Fusion

    Nexus Fusion is the web interface of Blue Brain Nexus, powered by Nexus Delta. It offers prebuilt data visualization, querying, data manipulation, and administration capabilities to help integrate, edit and visualize data and resources inside Nexus Delta projects.

    @@ -292,7 +299,7 @@

    diff --git a/docs/fusion/my-data.html b/docs/fusion/my-data.html index f6e4e1049a..96235698a1 100644 --- a/docs/fusion/my-data.html +++ b/docs/fusion/my-data.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -253,7 +253,7 @@
      @@ -281,6 +281,13 @@
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    User data

    The My Data page within the Nexus Fusion platform serves as a central repository for all datasets that have been created or updated by the logged in user. This feature enables users to easily access and manage the data that they are interested in, without the need to navigate through irrelevant data.

    @@ -301,7 +308,7 @@

    diff --git a/docs/fusion/organizations.html b/docs/fusion/organizations.html index 9e3d3e4de3..947daa0b3c 100644 --- a/docs/fusion/organizations.html +++ b/docs/fusion/organizations.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -250,7 +250,7 @@
      @@ -275,6 +275,13 @@
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    Organizations

    The Nexus Fusion platform provides users with the ability to manage a high-level entity that encompasses all projects within their ecosystem. This feature is accessible through the organizations page, which presents users with a comprehensive overview of all the organizations managed within a given Blue Brain Nexus deployment. Users who possess the necessary permissions are also able to create new organizations through this interface.

    @@ -293,7 +300,7 @@

    diff --git a/docs/fusion/plugins.html b/docs/fusion/plugins.html index 7e3cb0d6e6..3439904a50 100644 --- a/docs/fusion/plugins.html +++ b/docs/fusion/plugins.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -271,7 +271,7 @@
      @@ -317,6 +317,13 @@
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    Plugins

    A resource returned by the Nexus Delta API is usually a JSON-LD document. By default, Nexus Fusion displays JSON-LD in a code editor. If the user has edit right to the document, they can update the resource in the editor.

    @@ -443,7 +450,7 @@

    -v1.8.x +Snapshot

    diff --git a/docs/fusion/project.html b/docs/fusion/project.html index b319cd8661..9ceb26aa6b 100644 --- a/docs/fusion/project.html +++ b/docs/fusion/project.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -276,7 +276,7 @@
      @@ -327,6 +327,13 @@
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    Projects

    The concept of a Project in Nexus Fusion represents a fundamental unit of data management that empowers users to effectively organize and manage a distinct set of data. The Projects page, available in two distinct modes, enables users to browse either all projects within the Nexus ecosystem or those specific to a particular organization.

    @@ -266,7 +273,7 @@

    diff --git a/docs/fusion/search.html b/docs/fusion/search.html index 6d88467cf4..b0c0cc4d1c 100644 --- a/docs/fusion/search.html +++ b/docs/fusion/search.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -264,7 +264,7 @@
      @@ -303,6 +303,13 @@
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    Studio

    A Studio is a collection of persistent queries organized in a table layout for users to quickly access relevant data in a customizable way. Studio authors can create a studio to match a specific topic, and create organization schemes called Workspaces and Dashboards to access various aspects of that data.

    Note
    @@ -539,7 +546,7 @@

    diff --git a/docs/fusion/studios.html b/docs/fusion/studios.html index 4ba6547579..0a6ffadfbd 100644 --- a/docs/fusion/studios.html +++ b/docs/fusion/studios.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -243,7 +243,7 @@
      @@ -252,6 +252,13 @@
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    Studios

    The Studios space within the Nexus Fusion platform provides data curators with a powerful tool to showcase their data and can effectively visualize and communicate complex data sets to a broad audience. By developing custom plugins with JavaScript, curators can tailor the presentation of query results to meet specific requirements, including the formatting of charts, graphs, and other data visualization tools.

    @@ -265,7 +272,7 @@

    <

    diff --git a/docs/getting-started/index.html b/docs/getting-started/index.html index 1a5a968c01..8a5b836f2a 100644 --- a/docs/getting-started/index.html +++ b/docs/getting-started/index.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -243,7 +243,7 @@
      @@ -252,6 +252,13 @@
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    Getting Started

    The Nexus ecosystem is a data catalog with several technologies and techniques used to get insight and explore the data within the ecosystem.

    @@ -265,7 +272,7 @@

    diff --git a/docs/getting-started/notebooks/forge.yml b/docs/getting-started/notebooks/forge.yml index c9feb37edc..7e59396e79 100644 --- a/docs/getting-started/notebooks/forge.yml +++ b/docs/getting-started/notebooks/forge.yml @@ -12,13 +12,15 @@ Store: sparql: endpoint: "https://bluebrain.github.io/nexus/vocabulary/defaultSparqlIndex" vocabulary: - iri: "https://bluebrain.github.io/nexus/contexts/metadata.json" + metadata: + iri: "https://bluebrain.github.io/nexus/contexts/metadata.json" + local_iri: "https://bluebrainnexus.io/contexts/metadata.json" namespace: "https://bluebrain.github.io/nexus/vocabulary/" deprecated_property: "https://bluebrain.github.io/nexus/vocabulary/deprecated" project_property: "https://bluebrain.github.io/nexus/vocabulary/project" - max_connection: 50 + max_connection: 5 versioned_id_template: "{x.id}?rev={x._store_metadata._rev}" file_resource_mapping: https://raw.githubusercontent.com/BlueBrain/nexus-forge/master/examples/configurations/nexus-store/file-to-resource-mapping.hjson Formatters: - identifier: https://bbp.epfl.ch/neurosciencegraph/data/{}/{} \ No newline at end of file + identifier: https://bbp.epfl.ch/neurosciencegraph/data/{}/{} diff --git a/docs/getting-started/running-nexus/configuration/index.html b/docs/getting-started/running-nexus/configuration/index.html index 8c93d04280..decb042e61 100644 --- a/docs/getting-started/running-nexus/configuration/index.html +++ b/docs/getting-started/running-nexus/configuration/index.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    Nexus configuration

    -

    Nexus Delta service can be highly customized using configuration file(s). Many things can be adapted to your deployment needs: port where the service is running, timeouts, pagination defaults, etc.

    +

    Nexus Delta service can be highly customized using configuration file(s). Many things can be adapted to your deployment needs: port where the service is running, timeouts, pagination defaults, etc.

    There are 3 ways to modify the default configuration:

    • Setting the env variable DELTA_EXTERNAL_CONF which defines the path to a HOCON file. The configuration keys that are defined here can be overridden by the other methods.
    • @@ -337,24 +342,24 @@

      In terms of JVM pool memory allocation, we recommend setting the following values to the JAVA_OPTS environment variable: -Xms4g -Xmx4g. The recommended values should be changed accordingly with the usage of Nexus Delta, the number of projects and the resources/schemas size.

      In order to successfully run Nexus Delta there is a minimum set of configuration flags that need to be specified

      Http configuration

      -

      The http section of the configuration defines the binding address and port where the service will be listening.

      +

      The http section of the configuration defines the binding address and port where the service will be listening.

      The configuration flag akka.http.server.parsing.max-content-length can be used to control the maximum payload size allowed for Nexus Delta resources. This value applies to all posted resources except for files.

      Postgres configuration

      -

      The database section of the configuration defines the postgres specific configuration. As Nexus Delta uses three separate pools (‘read’, ‘write’, ‘streaming’), it is recommended to set the host, port, database name, username, and password via the app.defaults.database field, as it will apply to all pools. It is however possible to accommodate more advanced setups by configuring each pool separately by changing its respective app.database.{read|write|streaming} fields.

      +

      The database section of the configuration defines the postgres specific configuration. As Nexus Delta uses three separate pools (‘read’, ‘write’, ‘streaming’), it is recommended to set the host, port, database name, username, and password via the app.defaults.database field, as it will apply to all pools. It is however possible to accommodate more advanced setups by configuring each pool separately by changing its respective app.database.{read|write|streaming} fields.

      The pool size can be set using the app.defaults.database.access.pool-size setting for all pools, or individually for each pool (app.database.{read|write|streaming}.access.pool-size).

      Note

      A default Postgres deployment will limit the number of connections to 100, unless configured otherwise. See the Postgres Connection and Authentication documentation.

      -

      Before running Nexus Delta, the expected tables should be created.

      +

      Before running Nexus Delta, the init scripts should be run in the lexicographical order.

      It is possible to let Nexus Delta automatically create them using the following configuration parameters: app.database.tables-autocreate=true.

      Note

      Auto creation of tables is included as a development convenience and should be avoided in production.

      RDF parser

      -

      The underlying Apache Jena parser used to validate incoming data is configurable using the json-ld-api field to enable different levels of strictness.

      +

      The underlying Apache Jena parser used to validate incoming data is configurable using the json-ld-api field to enable different levels of strictness.

      Service account configuration

      Nexus Delta uses a service account to perform automatic tasks under the hood. Examples of it are:

      • Granting default ACLs to the user creating a project.
      • Creating default views on project creation.
      -

      The service-account section of the configuration defines the service account configuration.

      +

      The service-account section of the configuration defines the service account configuration.

      Automatic project provisioning

      Automatic project provisioning allows to create a dedicated project for users the first time they connect to Delta that is to say the first time, they query the project listing endpoints.

      The generated project label will be:

      @@ -363,49 +368,90 @@

      The automatic-provisioning section of the configuration defines the project provisioning configuration.

      -

      Encryption configuration

      -

      Nexus Delta uses symmetric encryption to secure sensitive data information (tokens and passwords).

      -

      The encryption section of the configuration defines the encryption configuration.

      +

      The automatic-provisioning section of the configuration defines the project provisioning configuration.

      Fusion configuration

      When fetching a resource, Nexus Delta allows to return a redirection to its representation in Fusion by providing text/html in the Accept header.

      -

      The fusion section of the configuration defines the fusion configuration.

      +

      The fusion section of the configuration defines the fusion configuration.

      Projections configuration

      Projections in Nexus Delta are asynchronous processes that can replay the event log and process this information. For more information on projections, please refer to the Architecture page.

      -

      The projections section of the configuration allows to configure the projections.

      +

      The projections section of the configuration allows to configure the projections.

      In case of failure in a projection, Nexus Delta records the failure information inside the public.failed_elem_logs PostgreSQL table, which can be used for analysis, and ultimately resolution of the failures. The configuration allows to set how long the failure information is stored for (app.projections.failed-elem-ttl), and how often the projection deleting the expired failures is awoken (app.projections.delete-expired-every).

      Plugins configuration

      Since 1.5.0, Nexus Delta supports plugins. Jar files present inside the local directory defined by the DELTA_PLUGINS environment variable are loaded as plugins into the Delta service.

      Each plugin configuration is rooted under plugins.{plugin_name}. All plugins have a plugins.{plugin_name}.priority configuration flag used to determine the order in which the routes are handled in case of collisions.

      For more information about plugins, please refer to the Plugins page.

      Elasticsearch views plugin configuration

      -

      The elasticsearch plugin configuration can be found here.

      +

      The elasticsearch plugin configuration can be found here.

      The most important flags are: * plugins.elasticsearch.base which defines the endpoint where the Elasticsearch service is running. * plugins.elasticsearch.credentials.username and plugins.elasticsearch.credentials.password to allow to access to a secured Elasticsearch cluster. The user provided should have the privileges to create/delete indices and read/index from them.

      Please refer to the Elasticsearch configuration which describes the different steps to achieve this.

      Blazegraph views plugin configuration

      -

      The blazegraph plugin configuration can be found here.

      +

      The blazegraph plugin configuration can be found here.

      The most important flag is plugins.blazegraph.base which defines the endpoint where the Blazegraph service is running.

      -

      The plugins.blazegraph.slow-queries section of the Blazegraph configuration defines what is considered a slow Blazegraph query, which will get logged in the public.blazegraph_queries PostgreSQL table. This can be used to understand which Blazegraph queries can be improved.

      +

      The plugins.blazegraph.slow-queries section of the Blazegraph configuration defines what is considered a slow Blazegraph query, which will get logged in the public.blazegraph_queries PostgreSQL table. This can be used to understand which Blazegraph queries can be improved.

      Composite views plugin configuration

      -

      The composite views plugin configuration can be found here.

      +

      The composite views plugin configuration can be found here.

      There are several configuration flags related to tweaking the range of values allowed for sources, projections and rebuild interval.

      +

      Authentication for remote sources can be specified in three different ways. The value of plugins.composite-views.remote-source-credentials should be speficied in the same way as remote storages, as shown here

      Storage plugin configuration

      -

      The storage plugin configuration can be found here.

      +

      The storage plugin configuration can be found here.

      Nexus Delta supports 3 types of storages: ‘disk’, ‘amazon’ (s3 compatible) and ‘remote’.

      • For disk storages the most relevant configuration flag is plugins.storage.storages.disk.default-volume, which defines the default location in the Nexus Delta filesystem where the files using that storage are going to be saved.
      • For S3 compatible storages the most relevant configuration flags are the ones related to the S3 settings: plugins.storage.storages.amazon.default-endpoint, plugins.storage.storages.amazon.default-access-key and plugins.storage.storages.amazon.default-secret-key.
      • -
      • For remote disk storages the most relevant configuration flags are plugins.storage.storages.remote-disk.default-endpoint (the endpoint where the remote storage service is running) and plugins.storage.storages.remote-disk.default-credentials (the Bearer token to authenticate to the remote storage service).
      • +
      • For remote disk storages the most relevant configuration flags are plugins.storage.storages.remote-disk.default-endpoint (the endpoint where the remote storage service is running) and plugins.storage.storages.remote-disk.credentials (the method to authenticate to the remote storage service).
      • +
      +

      File configuration

      +

      When the media type is not provided by the user, Delta relies on automatic detection based on the file extension in order to provide one.

      +

      From 1.9, it is possible to provide a list of extensions with an associated media type to compute the media type.

      +

      This list can be defined at files.media-type-detector.extensions:

      +
      files {
      +  # Allows to define default media types for the given file extensions
      +  media-type-detector {
      +    extensions {
      +      custom = "application/custom"
      +      ntriples = "application/n-triples"
      +    }
      +  }
      +}
      +
      +

      The media type resolution process follow this order stopping at the first successful step:

      +
        +
      • Select the Content-Type header from the file creation/update request
      • +
      • Compare the extension to the custom list provided in the configuratio
      • +
      • Fallback on akka automatic detection
      • +
      • Fallback to the default value application/octet-stream
      +

      Remote storage configuration

      +

      Authentication for remote storage can be specified in three different ways. The value of plugins.storage.storages.remote-disk.credentials can be:

      +
      Recommended: client credentials (OpenId authentication)
      +
      {
      +  type: "client-credentials"
      +  user: "username"
      +  password: "password"
      +  realm: "internal"
      +}
      +
      +

      This configuration tells Delta to log into the internal realm (which should have already been defined) with the user and password credentials, which will give Delta an access token to use when making requests to remote storage

      +
      Anonymous
      +
      {
      +  type: "anonymous"
      +}
      +
      +
      Long-living auth token (legacy)
      +
      {
      +  type: "jwt-token"
      +  token: "long-living-auth-token"
      +}
      +

      Archive plugin configuration

      -

      The archive plugin configuration can be found here.

      +

      The archive plugin configuration can be found here.

      Jira plugin configuration

      -

      The Jira plugin configuration can be found here.

      +

      The Jira plugin configuration can be found here.

      Setting up the Jira plugin needs to set up the endpoint of the Jira instance but also the consumer key, the consumer secret and the private key required to interact with Jira (more details including the configuration steps in Jira here).

      Monitoring

      For monitoring, Nexus Delta relies on Kamon.

      Kamon can be disabled by passing the environment variable KAMON_ENABLED set to false.

      -

      Delta configuration for Kamon is provided in the monitoring section. For a more complete description on the different options available, please look at the Kamon website.

      +

      Delta configuration for Kamon is provided in the monitoring section. For a more complete description on the different options available, please look at the Kamon website.

      Instrumentation

      Delta provides the Kamon instrumentation for:

    diff --git a/docs/getting-started/running-nexus/docker/docker-compose.yaml b/docs/getting-started/running-nexus/docker/docker-compose.yaml index dd2fa5dd8a..0b665523aa 100644 --- a/docs/getting-started/running-nexus/docker/docker-compose.yaml +++ b/docs/getting-started/running-nexus/docker/docker-compose.yaml @@ -5,7 +5,7 @@ services: - blazegraph - elasticsearch - postgres - image: bluebrain/nexus-delta:1.8.0-M10 + image: bluebrain/nexus-delta:1.9.0-M10 environment: DELTA_PLUGINS: "/opt/docker/plugins/" DELTA_EXTERNAL_CONF: "/config/delta.conf" @@ -23,7 +23,7 @@ services: memory: 1024M elasticsearch: - image: "docker.elastic.co/elasticsearch/elasticsearch:8.7.1" + image: "docker.elastic.co/elasticsearch/elasticsearch:8.11.0" environment: discovery.type: "single-node" bootstrap.memory_lock: "true" @@ -38,7 +38,7 @@ services: memory: 512M postgres: - image: library/postgres:15.3 + image: library/postgres:15.5 environment: POSTGRES_USER: "postgres" POSTGRES_PASSWORD: "postgres" @@ -57,7 +57,7 @@ services: web: depends_on: - delta - image: bluebrain/nexus-web:1.8.0 + image: bluebrain/nexus-web:1.9.9 environment: BASE_PATH: "/" HOST_NAME: "http://localhost" diff --git a/docs/getting-started/running-nexus/index.html b/docs/getting-started/running-nexus/index.html index aef9be2610..9f44c10aa7 100644 --- a/docs/getting-started/running-nexus/index.html +++ b/docs/getting-started/running-nexus/index.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -278,7 +278,7 @@
      @@ -331,6 +331,13 @@
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +
    diff --git a/docs/getting-started/running-nexus/search-configuration.html b/docs/getting-started/running-nexus/search-configuration.html index a93338cc44..49980cd815 100644 --- a/docs/getting-started/running-nexus/search-configuration.html +++ b/docs/getting-started/running-nexus/search-configuration.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -251,7 +251,7 @@
      @@ -277,9 +277,16 @@
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    Search configuration

    -

    Nexus provides global search functionality across all projects through the search plugin.

    Warning
    +

    Nexus provides global search functionality across all projects through the search plugin.

    Warning

    The search plugin is experimental and its functionality and API can change without notice.

    There are several aspects that have been taken into consideration when adding global search capabilities in Nexus:

    diff --git a/docs/getting-started/try-nexus-movielens.html b/docs/getting-started/try-nexus-movielens.html index 2858f11181..7ad5d7d9aa 100644 --- a/docs/getting-started/try-nexus-movielens.html +++ b/docs/getting-started/try-nexus-movielens.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -258,7 +258,7 @@
      @@ -291,6 +291,13 @@
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    Try Nexus with the MovieLens Dataset

    In this tutorial, you will use the core features of the Nexus ecosystem through our sandbox. This requires minimal technical knowledge but the ability to install a Python library and run a jupyter notebook.

    @@ -331,9 +338,9 @@

    MovieLens dataset into the created project within Nexus Delta using the python framework Nexus Forge.

    A jupyter notebook is available for this part of the tutorial and can be spawn easily using Google Colab, binder, or locally:

    For local execution, Nexus Forge can be installed using these instructions. Make sure that the jupyter notebook|lab is launched in the same virtual environment where Nexus Forge is installed. Alternatively, set up a specialized kernel.

    If you want to try some other examples of Nexus Forge, you can use these notebooks.

    @@ -381,8 +388,8 @@

    Github -
  • Google Colab
  • +
  • Github
  • +
  • Google Colab
  • Querying a Knowledge Graph using Elasticsearch

    The goal of this notebook is to learn how to connect to an Elasticsearch view and run queries against it.

    @@ -392,8 +399,8 @@

    Github -
  • Google Colab
  • +
  • Github
  • +
  • Google Colab
  • Linking data on the web

    In this tutorial, we demonstrate how to consume structured data published on the web according to the Linked data principles to extend and enrich a knowledge graph.

    @@ -409,8 +416,8 @@

    Github -
  • Google Colab
  • +
  • Github
  • +
  • Google Colab
  • @@ -420,7 +427,7 @@

    -v1.8.x +Snapshot

    diff --git a/docs/getting-started/try-nexus.html b/docs/getting-started/try-nexus.html index 6679efe51b..f3407e82ef 100644 --- a/docs/getting-started/try-nexus.html +++ b/docs/getting-started/try-nexus.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -303,7 +303,7 @@
      @@ -381,6 +381,13 @@
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    Try Nexus with Neuroscience Datasets

    Welcome to our Nexus tutorial! Nexus is an open-source data and metadata management suite. With Nexus, your data is catalogued and indexed as a knowledge graph, all interconnected.

    @@ -392,7 +399,7 @@

    edX EPFL Simulation Neuroscience MOOC.

    +

    This tutorial is also part of the edX EPFL Simulation Neuroscience MOOC.

    In Step 0, you’ll learn to setup your Python environment if you want to run the Jupyter notebooks locally. You can skip this step if you use Binder or Google Colaboratory (preferred).

    In Step 1, you’ll learn about the Nexus Sandbox deployment, a dedicated environment for this tutorial. You’ll use Nexus Fusion to login and access your dedicated project instantiated in Nexus Delta.

    @@ -494,11 +501,11 @@

    Read more about mapping with Forge.

    -

    In the code available, we will integrate data from two sources: the AIBS and MouseLight (see Step 3). We will provide mappers for both data sources. You can check the mappers directly on GitHub. There are two mappers for the AIBS, one for neuron morphologies and another for electrophysiology data. For MouseLight, there’s only one mapper, as both morphologies and traces are in the same data source.

    +

    In the code available, we will integrate data from two sources: the AIBS and MouseLight (see Step 3). We will provide mappers for both data sources. You can check the mappers directly on GitHub. There are two mappers for the AIBS, one for neuron morphologies and another for electrophysiology data. For MouseLight, there’s only one mapper, as both morphologies and traces are in the same data source.

    2.6. Running the Notebook

    To run the notebook locally, open your terminal, clone the Nexus repository, go to the notebook directory, and run Jupyter:

    @@ -533,8 +540,8 @@

    The example notebooks will use these endpoints to collect and download the datasets and their metadata.

    3.2. Running the Notebook

    To run the notebook locally, open your terminal, clone the Nexus repository, go to the notebook directory, and run Jupyter:

    @@ -937,8 +944,8 @@

    5.2. Running the Notebook

    To run the notebook locally, open your terminal, clone the Nexus repository, go to the notebook directory, and run Jupyter:

    @@ -954,7 +961,7 @@

    jupyter notebook MOOC_Content_based_Recommender_System_using_Blue_Brain_Nexus.ipynb

    Well done! You have now completed the last part of this tutorial. To learn more, scroll down or navigate to our documentation, or start contributing to our Github repositories.

    -

    If you have reached this tutorial via the Simulation Neuroscience MOOC, you can now head back to the edX platform and complete this week assignment.

    +

    If you have reached this tutorial via the Simulation Neuroscience MOOC, you can now head back to the edX platform and complete this week assignment.

    Learn More

    Another Tutorial with the MovieLens Dataset

    Nexus can be used to manage more than neuroscience data. If you want to try it, head over to our MovieLens Tutorial!

    @@ -966,7 +973,7 @@

    -v1.8.x +Snapshot

    diff --git a/docs/getting-started/understanding-knowledge-graphs.html b/docs/getting-started/understanding-knowledge-graphs.html index 9a562cca4c..ea58ff1add 100644 --- a/docs/getting-started/understanding-knowledge-graphs.html +++ b/docs/getting-started/understanding-knowledge-graphs.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -264,7 +264,7 @@
      @@ -303,6 +303,13 @@
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    Understanding Knowledge Graphs

    This section will help you understand knowledge graphs and related standards and technologies.

    @@ -424,7 +431,7 @@

    diff --git a/docs/index.html b/docs/index.html index 303dcf6f5b..73ad65faad 100644 --- a/docs/index.html +++ b/docs/index.html @@ -149,6 +149,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -197,11 +192,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -254,7 +254,7 @@
      @@ -288,6 +288,13 @@
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    Blue Brain Nexus

    Blue Brain Nexus is an ecosystem that allows you to organize and better leverage your data through the use of a Knowledge Graph. In addition to the products listed here, you’ll find a rich ecosystem of libraries and tools.

    @@ -325,7 +332,7 @@

    <

    diff --git a/docs/releases/index.html b/docs/releases/index.html index fbf6eeddfc..012c433f0e 100644 --- a/docs/releases/index.html +++ b/docs/releases/index.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    Releases

    This section of the documentation lists the significant BlueBrain Nexus releases across all services and web applications.

    The latest stable release is v1.8.0 released on 14.06.2023.

    +

    1.9.0

    +

    Breaking changes

    + +

    New features / enhancements

    +

    1.8.0 (14.06.2023)

    Breaking changes

    A detailed list of changes included in the release can be found in the release notes.

    @@ -591,152 +604,8 @@

    ElasticSearch views can be configured with settings (this allows the customization of ElasticSearch index with number of shards and replicas, tokenizers, filters etc.).

    A detailed list of changes included in the release can be found in the release notes.

    -

    v1.4.2 (20.10.2020)

    -

    This is a bugfix release, addressing the following issues related to views lifecycle:

    -
      -
    • Persist project statistics to avoid starting from NoOffset when service restarts or view collapses.
    • -
    • Prevent a deprecated organization/project from starting its views.
    • -
    -

    v1.4.1 (24.08.2020)

    -

    This is a bugfix release, addressing the following issues:

    -
      -
    • Project tag reported as a metric with Kamon is inconsistent
    • -
    • File Attribute computation is no longer exposed as a metric
    • -
    • The path prefix read from the app.http.public-uri is applied twice for KG specific routes
    • -
    • Parallelize the v1.3.x to v1.4.x migration script
    • -
    • Support a retry mechanism for the v1.3.x to v1.4.x migration
    • -
    • Add email address in footer
    • -
    • Add CONP and SWITCH logos on product page
    • -
    • Add SEO metadata to product pages
    • -
    • Add SEO headers
    • -
    -

    v1.4.0 (14.08.2020)

    -

    The release is backwards compatible with v1.x.y releases in terms of API. If you’re upgrading from v1.3.x please visit the migration instructions.

    -

    Summary of the significant changes:

    -
      -
    • Merged iam, admin and kg services into a single service, called delta;
    • -
    • Listings API now shows - besides resources metadata - the following predicates, when present: sko:prefLabel, schema: name, rdfs:label;
    • -
    • Nexus Web has evolved into Nexus Fusion, supporting multiple subapps and making the different sections clear for our users;
    • -
    • Greatly improved design for the way Nexus Fusion manages plugins;
    • -
    • Introduction of Nexus Forge in the ecosystem. Nexus Forge is currently at version 0.3.3.
    • -
    -

    A detailed list of changes included in the release can be found in the release notes.

    -

    v1.3.0 (25.02.2020)

    -

    The release is backwards compatible with v1.x.y releases in terms of API. If you’re upgrading from v1.2.x please visit the migration instructions.

    -

    Summary of the significant changes:

    -
      -
    • Introduced a new type of view (_CompositeView_, currently as a Beta feature) that expands on the indexing capabilities of the system through the ability to consume multiple sources (multiple projects in the same Nexus deployment and projects in different Nexus deployments);
    • -
    • Added the ability to generate tabular views on the data available to a specific project (using any SparqlView defined in the project - default SparqlView or AggregateSparqlViews) by means of Studios and Dashboards in Nexus Web;
    • -
    • Allow querying SparqlViews using the GET http method;
    • -
    • Exposed a new view subresource .../offset that presents the current view offset, or collection of offsets in case of CompositeViews. The offset has the same value used with Server Sent Events as means of keeping track of the current event replay progress. Deleting this resource with instruct the system to rebuild the indices of the selected view;
    • -
    • Ordering results when doing listings can now be controlled with the repeated sort query param that accepts ElasticSearch document field names (...?sort=_createdAt&sort=-_createdBy). The ordering defaults to ascending, but can be switched for descending by prefixing the - character to the field name.
    • -
    • New ElasticSearch indices are automatically configured to perform word split and properly handle UUIDs. The new configuration yields better full text search results.
    • -
    • Nexus Web - Adds the ability to have persistent customisable queries and data visualizations for your data via the new Studios feature
    • -
    -

    A detailed list of changes included in the release can be found in the release notes.

    -

    v1.2.1 (07.10.2019)

    -

    This is a bugfix release, addressing two specific issues:

    -
      -
    • Fix FileAttributesUpdated event deserialization which was causing indexing problems for situations where the deployment included a remote storage service to handle files.
    • -
    • Removed kamon-akka-remote dependency which was causing problems in clustered deployments due to binary compatibility issues.
    • -
    -

    v1.2.0 (04.10.2019)

    -

    The release adds two major features:

    -
      -
    • endpoint to fetch the original payload of a resource.
    • -
    • ability to retrieve multiple resources in one request as an archive.
    • -
    -

    The API is backwards compatible with v1.1.

    -

    Summary of the significant changes:

    -

    Storage service related updates:

    -
      -
    • Updated async computation of to return not only the digest information but all the attributes (bytes, digest, mediaType and location).
    • -
    -

    KG updates:

    -
      -
    • Added archives resources.
    • -
    • Added /source sub-resource.
    • -
    • Fixed issue with resource retrieval when linked context changes.
    • -
    • Updated DigestViewCoordinator to AttributesViewCoordinator. This async process now updates all the FileAttributes.
    • -
    -

    Dependency updates:

    -
      -
    • SHACL validator, akka-http, cats, cats-effects amongst others
    • -
    -

    v1.1.2 (24.09.2019)

    -

    The release addresses bug fixing and is backwards compatible with v1.0 in terms of API. If you’re upgrading from v1.0 please visit the migration instructions.

    -

    Summary of the significant changes:

    -

    Storage service related updates:

    -
      -
    • Added async computation of the file digest.
    • -
    • Before an action gets executed against the storage, checks that the resource created is valid (is not deprecated, has the correct revision, etc…)
    • -
    -

    KG Fixes:

    -
      -
    • When project is not present in the cache but it is present in the underlying admin service, adds it directly to the cache (before the cache was populated from the SSE, which can be very slow).
    • -
    • ProjectViewCoordinator and DigestViewCoordinator actors now create child actors (better management of actors lifecycle).
    • -
    • Prevented from creating unnecessary indices/namespaces.
    • -
    -

    Fixed library dependency issues:

    -
      -
    • Corrected Iri to Akka.Uri conversion
    • -
    • Corrected pct encoding (Iri.asString and Iri.asUri)
    • -
    • Bumped akka and kamon dependencies, amongst others
    • -
    -

    v1.1.1 (24.07.2019)

    -

    The release addresses bug fixing and is backwards compatible with v1.0 in terms of API. If you’re upgrading from v1.0 please visit the migration instructions.

    -

    Summary of the significant changes:

    -
      -
    • Migration script correctly updates views with the expected defaults
    • -
    • Migration script jumps over event deserialization errors
    • -
    • Metric tag value fix for elasticsearch indexer
    • -
    • Kamon disabled by default
    • -
    • Kamon agent is loaded as a JVM argument
    • -
    • Updated library dependencies
    • -
    -

    v1.1 (19.07.2019)

    -

    The release is backwards compatible with v1.0 in terms of API. If you’re upgrading from v1.0 please visit the migration instructions.

    -

    Summary of the significant changes:

    -
      -
    • Exposed the service event logs over a stable API via Server Sent Events.
    • -
    • Introduced configurable storage backends for files with local, remote and S3 implementations.
    • -
    • ElasticSearchView | AggregateElasticSearchView have been promoted to stable.
    • -
    • Introduced a new SPARQL view, AggregateSparqlView, that dispatches SPARQL queries to the appropriate namespaces and aggregates the results.
    • -
    • ElasticSearchView and SparqlView support additional configuration options: resourceSchemas, resourceTypes, resourceTag, includeDeprecated, includeMetadata.
    • -
    • API improvements: -
        -
      • Support for additional filtering criteria when listing resources via query params: rev, deprecated, createdBy, updatedBy.
      • -
      • The organization and project segments when exercising the API now accept their unique ids (UUID).
      • -
      • Content negotiation for resources, supporting: dot, ntriples, json-ld expanded and compacted formats.
      • -
      • Ability to resolve resource ids via configured project resolvers.
      • -
      • Pagination of resources over 10,000 using the ’‘_next’’ link in the listing response.
      • -
      • Resource metadata includes ’‘_incoming’‘ and ’‘_outgoing’’ links and the API now includes their respective endpoints.
      • -
      • View indexing progress as a ‘‘statistics’’ sub-resource of each view.
      • -
      -
    • -
    • Nexus Web improvements: -
        -
      • Better OpenIdConnect integration, ability to authenticate to multiple configured realms.
      • -
      • Ability to discriminate between Nexus specific resources and user managed resources.
      • -
      • Display the current ACLs and their source for the logged in user.
      • -
      • Ability to query user defined views.
      • -
      • Display the indexing progress for the active view.
      • -
      -
    • -
    • Exposed view indexing progress metrics for Prometheus.
    • -
    • Bumped ElasticSearch compatibility to 7.x.
    • -
    -

    A detailed list of changes included in the release can be found in the release notes.

    -

    v1.0 (25.01.2019)

    -

    This is the first major release of Blue Brain Nexus after almost two years of development.

    -

    Also referred to as “Nexus V1”, this initial release is our first big milestone in our quest to build a Knowledge Graph platform uniquely combining flexible graph database, powerful search engine and scalable data store to enable:

    -
      -
    • Easy unification and integration of fragmented and disparate data from heterogeneous domains to break data and metadata silos
    • -
    • Better data governance with the ability to specify and enforce organization’s best practices for data collection, storage and description through high quality metadata
    • -
    • Data lineage and provenance recording and description
    • -
    • FAIR (Findable, Accessible, Interoperable, Re-usable) data and metadata management
    • -
    -

    A detailed list of changes included in the release can be found in the release notes.

    +

    Older releases

    +

    The release notes of older versions are available here.

    @@ -768,13 +637,13 @@

    -
    +
    + +
    +
    +
    + +
    +
    +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +
    +
    +

    Older releases

    +

    For archival purposes, the release notes of older versions are available here.

    +
    + + +
    +
    +
    +
    + + +
    + + + + + + + + + diff --git a/docs/releases/v1.4-to-v1.5-migration.html b/docs/releases/v1.4-to-v1.5-migration.html index 7ec7271ddd..e4c28a17b5 100644 --- a/docs/releases/v1.4-to-v1.5-migration.html +++ b/docs/releases/v1.4-to-v1.5-migration.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -278,7 +278,7 @@
      @@ -331,6 +331,13 @@
  • +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    v1.4 To v1.5 Migration

    The v1.5 release of Delta includes a lot of improvements like storing the compacted/expanded forms of resources to get immutability and improve performance.

    @@ -356,7 +363,7 @@

    nodetool repair

    Create a materialized view on the messages table on the original keyspace

    -

    Create a materialized view on the messages table on the original keyspace delta_1_4:

    +

    Create a materialized view on the messages table on the original keyspace delta_1_4:

        CREATE MATERIALIZED VIEW {delta_1_4}.ordered_messages AS
             SELECT ser_manifest, ser_id, event
             FROM delta.messages
    @@ -722,7 +729,7 @@ 

    Restart in normal mode

    Once migration is completed and the previous checks have been performed, Delta can be restarted in normal mode.

    -

    The previous Delta instance can be stopped. After the restart, a complete reindexing of the data in elasticsearch/blazegraph will take place.

    +

    The previous Delta instance can be stopped. After the restart, a complete reindexing of the data in elasticsearch/blazegraph will take place.

    • Stop the Delta 1.4 instance ;
    • Delete all ElasticSearch indices: @@ -743,7 +750,7 @@

      Delta configuration

    • +
    • Increasing temporarily the delay in retry strategies can also help reduce the pressure on the platform. See: Delta configuration
    • Check the statistics views for different views, they should progressively tend to get to 0 remaining events.

    • Check the cluster stats endpoint in elasticsearch, the number of indices and docs should stabilize after a while.
    • @@ -768,7 +775,7 @@

    @@ -791,13 +798,13 @@

    -
    +
    +
    Snapshot version
    +

    +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

    +

    v1.5 Release Notes

    @@ -327,15 +334,15 @@

    <

    Plugins support painless extensibility and evolution of different features of the Nexus Delta API while keeping isolation and stability to the rest of the feature set.

    In order to adopt and validate this new design, we have identified a set features that have been implemented as plugins:

    If you’re interested into building your own plugin, please visit the plugins section.

    JSON-LD

    Note
    -

    Please read carefully this section. Some changes introduced might have breaking consequences on clients consuming the Nexus Delta API if, and only if, your client does not already use a JSON-LD aware library

    +

    Please read carefully this section. Some changes introduced might have breaking consequences on clients consuming the Nexus Delta API if, and only if, your client does not already use a JSON-LD aware library

    JSON-LD has been our primary representation format since the beginnings of Nexus Delta. It is a powerful format that allows clients to use simple JSON while it adds support for Linked Data, opening the door to work with knowledge graphs.

    Our previous JSON-LD support was, in some cases, inconsistent with the specification. We have fixed that, introducing the following changes:

    diff --git a/docs/releases/v1.5-to-v1.6-migration.html b/docs/releases/v1.5-to-v1.6-migration.html index 6e37667a18..1c8a2b0c16 100644 --- a/docs/releases/v1.5-to-v1.6-migration.html +++ b/docs/releases/v1.5-to-v1.6-migration.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -271,7 +271,7 @@
      @@ -317,6 +317,13 @@
      +
      +
      Snapshot version
      +

      +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

      +

      v1.6 Release Notes

      @@ -359,7 +366,7 @@

      Secure the Delta Elasticsearch client

      With 1.6, it is now possible to define credentials to query a secured Elasticsearch cluster: Elasticsearch configuration.

      Dedicated Blazegraph client for user-defined queries

      -

      User-defined queries could bring instability to Blazegraph because of their complexity. In 1.6, a dedicated client with a timeout which triggers a circuit breaker in Blazegraph has been introduced in order to solve that problem.

      +

      User-defined queries could bring instability to Blazegraph because of their complexity. In 1.6, a dedicated client with a timeout which triggers a circuit breaker in Blazegraph has been introduced in order to solve that problem.

      Synchronous indexing

      From 1.6, the different types of resources can be indexed directly after creation/modification without waiting for the background indexing process to pick it up.

      Graph analytics

      @@ -368,7 +375,7 @@

      Listings

      The different types of resources can now be listed at the organization level and within all the projects the current user has access to.

      Configurable RDF parser

      -

      The RDF parser allowing to validate incoming data can now be configured in order to be more or less strict.

      +

      The RDF parser allowing to validate incoming data can now be configured in order to be more or less strict.

      Automatic project provisioning

      When enabled, a dedicated project is created for the current user on its first access to the Nexus platform.

      How to enable and configure it is detailed here

      @@ -383,7 +390,7 @@

      Nexus Fusion

      Features

        -
      • Revamped Fusion search. Fusion search will now use delta search end point and will provide a improved UI with sorting, filtering, pagination etc.
      • +
      • Revamped Fusion search. Fusion search will now use delta search end point and will provide a improved UI with sorting, filtering, pagination etc.
      • Real-time updates. Fusion now uses ‘indexing=sync’ option in delta API. This means all the updates made through fusion will be indexed in delta synchronously.
      • Sub apps other than studio will be hidden in sidebar for anonymous users.
      • Fusion will show a warning message when projects are due for deletion.
      • @@ -402,7 +409,7 @@

        -v1.8.x +Snapshot

      diff --git a/docs/releases/v1.6-to-v1.7-migration.html b/docs/releases/v1.6-to-v1.7-migration.html index 068fe18e56..fca53dc0af 100644 --- a/docs/releases/v1.6-to-v1.7-migration.html +++ b/docs/releases/v1.6-to-v1.7-migration.html @@ -154,6 +154,8 @@
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
      @@ -292,7 +292,7 @@
      @@ -359,6 +359,13 @@
      +
      +
      Snapshot version
      +

      +You are browsing the docs for the snapshot version of Nexus, +the latest release is available here +

      +

      v1.7 To v1.8 Migration

      The v1.8 release of Delta introduces PostgreSQL as sole option for the Delta primary event store.

      @@ -453,12 +460,12 @@

      Create the PostgreSQL tables

      -

      Run the commands from the Delta 1.8.x schema.ddl file to create the tables necessary for Delta operations.

      +

      Run the commands from the Delta 1.8.x schema scripts to create the tables necessary for Delta operations.

      Elasticsearch and Blazegraph indices

      For the migration, there are two ways to make sure that the new Delta 1.8 indices do not overlap the ones from Delta 1.7 (for both Elasticsearch and Blazegraph):

      1. Delete all existing indices
      2. -
      3. Configure a different prefix for the new Delta 1.8 indices using the app.defaults.indexing.prefix field, which sets the default prefix value for both Elasticsearch and Blazegraph indices.
      4. +
      5. Configure a different prefix for the new Delta 1.8 indices using the app.defaults.indexing.prefix field, which sets the default prefix value for both Elasticsearch and Blazegraph indices.
      Note

      Elasticsearch defaults to a maximum of 1000 shards per node in the cluster. If you intend to keep both old and new indices for the time of the migration, ensure that you can double your amount of shards within this limit. It is also possible to raise this limit by configuring cluster.max_shards_per_node. In both cases, monitor your resource usage, and consider allocating more resources to Elasticsearch for the time of the migration.

      Blazegraph journal size

      Blazegraph stores data using a journaling system, which is append-only. Consider backing up your existing Blazegraph journal (by making a copy of the blazegraph.jnl file), and running with a clean instance of Blazegraph. This can allow a significant reduction in the size of the journal as only the latest state will be saved.

      @@ -713,7 +720,7 @@

    +

    Delta 1.8 contains several bug fixes and improvements that result in differences in the Blazegraph counts. As such it is expected to find differences in the migration_blazegraph_count table. Additionally, the default Blazegraph query timeout is 1 minute, so that the count may not be able to complete for large projects.

    The calls must be authenticated using the same service account as the one defined in Delta.

    Before running the tests, the following command needs to be run in your PostgreSQL instance:

    CREATE TABLE IF NOT EXISTS public.migration_resources_diff_offset(
    @@ -847,7 +854,7 @@ 

    diff --git a/docs/releases/v1.8-release-notes.html b/docs/releases/v1.8-release-notes.html index 24ebc7237b..fb56304d55 100644 --- a/docs/releases/v1.8-release-notes.html +++ b/docs/releases/v1.8-release-notes.html @@ -154,6 +154,8 @@

  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks
  • Releases
  • Roadmap
  • Nexus Fusion @@ -202,11 +197,15 @@
  • Permissions
  • Realms
  • Access Control Lists
  • +
  • User Permissions
  • Organizations
  • Projects
  • Quotas
  • Schemas
  • Resources
  • +
  • Trial
  • +
  • Multi fetch
  • +
  • Id Resolution
  • Resolvers
  • Views
  • +
  • Nexus Metadata
  • Plugins
  • Benchmarks