Merge pull request #1520 from zazuko/shared-dim-slow

feat: searchable shared dimensions

.changeset/clean-boxes-marry.md

@@ -0,0 +1,5 @@
+---
+"@cube-creator/shared-dimensions-api": minor
+---
+
+Improve loading speed of Shared Dimensions in UI (closes #1509)

.changeset/thick-clocks-ring.md

@@ -0,0 +1,5 @@
+---
+"@cube-creator/shared-dimensions-api": minor
+---
+
+Shared dimensions are now searchable and paginated in search results on dimension mapping and hierarchy screens (re #1509, closes #1481)

api.Dockerfile

@@ -3,6 +3,7 @@ FROM node:18-alpine AS builder
 
 WORKDIR /app
 ADD package.json yarn.lock ./
+ADD ./patches ./patches/
 ADD ./apis/core/package.json ./apis/core/
 ADD ./apis/errors/package.json ./apis/errors/
 ADD ./apis/shared-dimensions/package.json ./apis/shared-dimensions/

apis/core/bootstrap/shapes/dimension.ts

@@ -3,13 +3,10 @@ import { supportedLanguages } from '@cube-creator/core/languages'
 import { sparql, turtle } from '@tpluscode/rdf-string'
 import { dash, hydra, prov, rdf, rdfs, schema, sh, qudt, time, xsd } from '@tpluscode/rdf-ns-builders'
 import namespace from '@rdfjs/namespace'
-import $rdf from 'rdf-ext'
 import { lindasQueryTemplate } from '../lib/query'
 
 const sou = namespace('http://qudt.org/vocab/sou/')
 
-const sharedDimensionCollection = $rdf.namedNode('dimension/_term-sets')
-
 const unitsQuery = sparql`
 CONSTRUCT
   {
@@ -472,7 +469,14 @@ ${shape('dimension/shared-mapping-import')} {
       ${sh.path} ${cc.sharedDimension} ;
       ${sh.name} "Shared dimension" ;
       ${dash.editor} ${dash.AutoCompleteEditor} ;
-      ${hydra.collection} ${sharedDimensionCollection} ;
+      ${hydra.search} [
+        ${hydra.template} "dimension/_term-sets?{&q}" ;
+        ${hydra.mapping} [
+          ${hydra.variable} "q" ;
+          ${hydra.property} ${hydra.freetextQuery} ;
+          ${sh.minLength} 0 ;
+        ];
+      ] ;
       ${sh1.orderBy} ( ${schema.name} ) ;
       ${sh.nodeKind} ${sh.IRI} ;
       ${sh.order} 10 ;
@@ -520,7 +524,14 @@ ${shape('dimension/shared-mapping')} {
       ${sh.path} ${cc.sharedDimension} ;
       ${sh.name} "Shared dimensions" ;
       ${dash.editor} ${dash.AutoCompleteEditor} ;
-      ${hydra.collection} ${sharedDimensionCollection} ;
+      ${hydra.search} [
+        ${hydra.template} "dimension/_term-sets?{&q}" ;
+        ${hydra.mapping} [
+          ${hydra.variable} "q" ;
+          ${hydra.property} ${hydra.freetextQuery} ;
+          ${sh.minLength} 0 ;
+        ];
+      ] ;
       ${sh1.orderBy} ( ${schema.name} ) ;
       ${sh.nodeKind} ${sh.IRI} ;
       ${sh.order} 10 ;

apis/shared-dimensions/bootstrap/entrypoint.ts

@@ -5,5 +5,5 @@ import type { BootstrappedResourceFactory } from './index'
 
 export const entrypoint = (ptr: BootstrappedResourceFactory, ns: NamespaceBuilder) =>
   ptr('').addOut(rdf.type, [hydra.Resource, md.Entrypoint])
-    .addOut(md.sharedDimensions, ns('_term-sets'))
+    .addOut(md.sharedDimensions, ns('_term-sets?pageSize=1000'))
     .addOut(md.hierarchies, ns('_hierarchies'))

apis/shared-dimensions/hydra/index.ttl

@@ -48,6 +48,27 @@ md:SharedDimensions
           a code:EcmaScript ;
           code:link <file:handlers/shared-dimensions#get> ;
         ] ;
+      hydra-box:variables
+        [
+          a hydra:IriTemplate ;
+          hydra:template "/_term-sets{?q,pageSize,page}" ;
+          hydra:mapping
+            [
+              a hydra:IriTemplateMapping ;
+              hydra:property hydra:freetextQuery ;
+              hydra:variable "q" ;
+            ],
+            [
+              a hydra:IriTemplateMapping ;
+              hydra:property hydra:limit ;
+              hydra:variable "pageSize" ;
+            ],
+            [
+              a hydra:IriTemplateMapping ;
+              hydra:property hydra:pageIndex ;
+              hydra:variable "page" ;
+            ] ;
+        ] ;
     ], [
          a hydra:Operation, schema:CreateAction ;
          hydra:method "POST" ;

apis/shared-dimensions/lib/domain/shared-dimensions.ts

@@ -1,8 +1,6 @@
 import path from 'path'
 import type { Quad, Stream, Term } from '@rdfjs/types'
-import { CONSTRUCT } from '@tpluscode/sparql-builder'
 import { hydra, rdf, schema, sh } from '@tpluscode/rdf-ns-builders'
-import { md, meta } from '@cube-creator/core/namespace'
 import $rdf from 'rdf-ext'
 import { toRdf } from 'rdf-literal'
 import { fromFile } from 'rdf-utils-fs'
@@ -11,28 +9,31 @@ import { isGraphPointer } from 'is-graph-pointer'
 import { StreamClient } from 'sparql-http-client/StreamClient'
 import { ParsingClient } from 'sparql-http-client/ParsingClient'
 import env from '../env'
-import shapeToQuery from '../shapeToQuery'
+import shapeToQuery, { rewriteTemplates } from '../shapeToQuery'
 import { getDynamicProperties } from './shared-dimension'
 
-export function getSharedDimensions() {
-  return CONSTRUCT`
-    ?termSet ?p ?o .
-    ?termSet ${md.terms} ?terms .
-    ?termSet ${md.export} ?export .
-  `
-    .WHERE`
-      ?termSet a ${schema.DefinedTermSet}, ${meta.SharedDimension} .
-      ?termSet ?p ?o .
-
-      MINUS { ?termSet ${md.export} ?o }
-
-      BIND ( IRI(CONCAT("${env.MANAGED_DIMENSIONS_BASE}", "dimension/_terms?dimension=", ENCODE_FOR_URI(STR(?termSet)))) as ?terms )
-
-      OPTIONAL {
-        ?termSet a ${md.SharedDimension} .
-        BIND ( IRI(CONCAT("${env.MANAGED_DIMENSIONS_BASE}", "dimension/_export?dimension=", ENCODE_FOR_URI(STR(?termSet)))) as ?export )
-      }
-    `
+interface GetSharedDimensions {
+  freetextQuery?: string
+  limit?: number
+  offset?: number
+}
+
+export async function getSharedDimensions<C extends StreamClient | ParsingClient>(client: C, { freetextQuery = '', limit = 10, offset = 0 }: GetSharedDimensions = {}): Promise<C extends StreamClient ? Stream : Quad[]> {
+  const { constructQuery } = await shapeToQuery()
+
+  const shape = await loadShape('dimensions-query-shape')
+
+  const { MANAGED_DIMENSIONS_BASE } = env
+  const variables = new Map(Object.entries({
+    MANAGED_DIMENSIONS_BASE,
+    limit,
+    offset,
+    freetextQuery,
+    orderBy: schema.name,
+  }))
+  await rewriteTemplates(shape, variables)
+
+  return constructQuery(shape).execute(client) as any
 }
 
 interface GetSharedTerms {
@@ -44,10 +45,7 @@ interface GetSharedTerms {
 }
 
 export async function getSharedTerms<C extends StreamClient | ParsingClient>({ sharedDimensions, freetextQuery, validThrough, limit = 10, offset = 0 }: GetSharedTerms, client: C): Promise<C extends StreamClient ? Stream : Quad[]> {
-  const shape = await loadShape()
-  if (!isGraphPointer(shape)) {
-    throw new Error('Multiple shapes found')
-  }
+  const shape = await loadShape('terms-query-shape')
 
   shape.addOut(sh.targetNode, sharedDimensions)
 
@@ -79,13 +77,19 @@ export async function getSharedTerms<C extends StreamClient | ParsingClient>({ s
   }
 
   const { constructQuery } = await shapeToQuery()
-  return constructQuery(shape).execute(client.query) as any
+  return constructQuery(shape).execute(client) as any
 }
 
-async function loadShape() {
-  const dataset = await $rdf.dataset().import(fromFile(path.resolve(__dirname, '../shapes/terms-query-shape.ttl')))
+async function loadShape(shape: string) {
+  const dataset = await $rdf.dataset().import(fromFile(path.resolve(__dirname, `../shapes/${shape}.ttl`)))
 
-  return clownface({
+  const ptr = clownface({
     dataset,
   }).has(rdf.type, sh.NodeShape)
+
+  if (!isGraphPointer(ptr)) {
+    throw new Error('Multiple shapes found')
+  }
+
+  return ptr
 }

apis/shared-dimensions/lib/handlers/collection.ts

@@ -1,23 +1,32 @@
 import type { NamedNode, Quad } from '@rdfjs/types'
 import $rdf from 'rdf-ext'
-import clownface from 'clownface'
+import clownface, { GraphPointer } from 'clownface'
 import { hydra, rdf } from '@tpluscode/rdf-ns-builders'
 
 interface CollectionHandler {
   memberType: NamedNode
   collectionType: NamedNode
+  view?: NamedNode
   memberQuads: Quad[]
   collection: NamedNode
 }
 
-export function getCollection({ collection, memberQuads, memberType, collectionType }: CollectionHandler) {
+export function getCollection({ collection, view, memberQuads, memberType, collectionType }: CollectionHandler): GraphPointer<NamedNode> {
   const dataset = $rdf.dataset(memberQuads)
 
   const graph = clownface({ dataset })
   const members = graph.has(rdf.type, memberType)
 
-  return graph.node(collection)
+  graph.node(collection)
     .addOut(rdf.type, [hydra.Collection, collectionType])
     .addOut(hydra.member, members)
     .addOut(hydra.totalItems, members.terms.length)
+
+  if (view) {
+    graph.node(view)
+      .addOut(rdf.type, hydra.PartialCollectionView)
+      .addIn(hydra.view, collection)
+  }
+
+  return graph.node(collection)
 }

apis/shared-dimensions/lib/handlers/shared-dimensions.ts

@@ -19,9 +19,24 @@ import { rewrite, rewriteTerm } from '../rewrite'
 import { postImportedDimension } from './shared-dimension/import'
 import { getCollection } from './collection'
 
-export const get = asyncMiddleware(async (req, res) => {
-  const collection = await getCollection({
-    memberQuads: await getSharedDimensions().execute(parsingClient.query),
+export const get = asyncMiddleware(async (req, res, next) => {
+  if (!req.dataset) {
+    return next(new httpError.BadRequest())
+  }
+  const query = clownface({ dataset: await req.dataset() })
+  const pageSize = Number(query.out(hydra.limit).value || 10)
+  const page = Number(query.out(hydra.pageIndex).value || 1)
+  const offset = (page - 1) * pageSize
+  const queryParams = {
+    freetextQuery: query.has(hydra.freetextQuery).out(hydra.freetextQuery).value,
+    validThrough: query.has(md.onlyValidTerms, query.literal(true)).terms.length ? new Date() : undefined,
+    limit: pageSize,
+    offset,
+  }
+
+  const collection = getCollection({
+    view: $rdf.namedNode(req.absoluteUrl()),
+    memberQuads: await getSharedDimensions(parsingClient, queryParams),
     collectionType: md.SharedDimensions,
     memberType: schema.DefinedTermSet,
     collection: req.hydra.resource.term,
@@ -73,7 +88,7 @@ export const getTerms = asyncMiddleware(async (req, res, next) => {
     offset,
   }
 
-  const collection = await getCollection({
+  const collection = getCollection({
     memberQuads: await getSharedTerms(queryParams, parsingClient),
     memberType: schema.DefinedTerm,
     collectionType: md.SharedDimensionTerms,

apis/shared-dimensions/lib/shapeToQuery.ts

@@ -1,11 +1,14 @@
 import onetime from 'onetime'
 import { md } from '@cube-creator/core/namespace'
-import { GraphPointer } from 'clownface'
+import { AnyPointer, GraphPointer } from 'clownface'
 import { isGraphPointer } from 'is-graph-pointer'
 import { hydra } from '@tpluscode/rdf-ns-builders'
 import { Parameters, PropertyShape } from '@hydrofoil/shape-to-query/model/constraint/ConstraintComponent'
+import evalTemplateLiteral from 'rdf-loader-code/evalTemplateLiteral.js'
 import namespace from '@rdfjs/namespace'
 import { sparql } from '@tpluscode/sparql-builder'
+import $rdf from 'rdf-ext'
+import type { Literal } from '@rdfjs/types'
 import env from './env'
 
 /*
@@ -15,23 +18,63 @@ import env from './env'
 // eslint-disable-next-line no-new-func
 const _importDynamic = new Function('modulePath', 'return import(modulePath)')
 
-export default async function shapeToQuery() {
+export default async function shapeToQuery(): Promise<Pick<typeof import('@hydrofoil/shape-to-query'), 'constructQuery' | 'deleteQuery' | 's2q'>> {
   await setup()
 
-  const { constructQuery, deleteQuery } = await _importDynamic('@hydrofoil/shape-to-query')
+  const { constructQuery, deleteQuery, s2q } = await _importDynamic('@hydrofoil/shape-to-query')
 
   return {
     constructQuery,
     deleteQuery,
+    s2q,
   }
 }
 
+export async function rewriteTemplates(shape: AnyPointer, variables: Map<string, unknown>) {
+  const { s2q } = await shapeToQuery()
+
+  shape.any().has(s2q('template' as any))
+    .forEach(templateNode => {
+      const template = templateNode.out(s2q('template' as any))
+      if (!isGraphPointer(template)) {
+        throw new Error('Template not found')
+      }
+
+      const value = evalTemplateLiteral(template.value, { variables })
+      const literalOption = (template.term as Literal).language || (template.term as Literal).datatype
+
+      ;[...shape.dataset.match(null, null, templateNode.term)].forEach(quad => {
+        shape.dataset.delete(quad)
+        shape.dataset.add($rdf.quad(quad.subject, quad.predicate, $rdf.literal(value, literalOption), quad.graph))
+      })
+
+      templateNode.deleteOut()
+    })
+
+  shape.any().has(s2q('variable' as any))
+    .forEach(templateNode => {
+      const variableName = templateNode.out(s2q('variable' as any)).value
+      if (!variableName) {
+        return
+      }
+
+      const value = variables.get(variableName) as any
+
+      ;[...shape.dataset.match(null, null, templateNode.term)].forEach(quad => {
+        shape.dataset.delete(quad)
+        shape.dataset.add($rdf.quad(quad.subject, quad.predicate, value, quad.graph))
+      })
+
+      templateNode.deleteOut()
+    })
+}
+
 const setup = onetime(async () => {
   await defineConstraintComponents()
 })
 
 async function defineConstraintComponents() {
-  const { ConstraintComponent } = await _importDynamic('@hydrofoil/shape-to-query/model/constraint/ConstraintComponent.js')
+  const { default: ConstraintComponent } = await _importDynamic('@hydrofoil/shape-to-query/model/constraint/ConstraintComponent.js')
   const { constraintComponents } = await _importDynamic('@hydrofoil/shape-to-query/model/constraint/index.js')
   const { PatternConstraintComponent } = await _importDynamic('@hydrofoil/shape-to-query/model/constraint/pattern.js')