Merge pull request #1533 from zazuko/ods-resources

opendata.swiss resources
zazuko · Sep 24, 2024 · 11082f2 · 11082f2
2 parents aa9a922 + 4ba35b3
commit 11082f2
Show file tree

Hide file tree

Showing 18 changed files with 183 additions and 41 deletions.
diff --git a/.changeset/bright-hairs-serve.md b/.changeset/bright-hairs-serve.md
@@ -0,0 +1,5 @@
+---
+"@cube-creator/ui": patch
+---
+
+Increase the default width of metadata panel
diff --git a/.changeset/fresh-olives-refuse.md b/.changeset/fresh-olives-refuse.md
@@ -0,0 +1,5 @@
+---
+"@cube-creator/ui": patch
+---
+
+Improve rendering of translated form fields to fill available space
diff --git a/.changeset/ten-clocks-shop.md b/.changeset/ten-clocks-shop.md
@@ -0,0 +1,5 @@
+---
+"@cube-creator/core-api": minor
+---
+
+Added resources to Opendata.swiss cube metadata (`dcat:Distribution`). re https://gitlab.ldbar.ch/bafu/visualize/-/issues/560
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -7,15 +7,21 @@ on:
 jobs:
   unit-test:
     runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        command:
+          - test
+          - test:queries
+          - test:cli:rest
     timeout-minutes: 45
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/workflows/setup-env
-      - run: yarn test
-      - run: yarn test:queries
-      - run: yarn test:cli:rest
+      - run: yarn ${{ matrix.command }}
         env:
           AUTH_RUNNER_CLIENT_SECRET: ${{ secrets.AUTH_RUNNER_CLIENT_SECRET }}
+          NODE_TLS_REJECT_UNAUTHORIZED: 0
       - name: logs on fail
         if: ${{ failure() }}
         run: lando logs -s core
@@ -47,7 +53,7 @@ jobs:
       - run: yarn seed-data
 
       - name: Hydra e2e tests
-        run: docker-compose -f docker-compose.yml -f docker-compose.posix.yml run e2e-tests
+        run: docker compose -f docker-compose.yml -f docker-compose.posix.yml run e2e-tests
       - name: core logs on fail
         if: ${{ failure() }}
         run: lando logs -s core
@@ -86,6 +92,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/workflows/setup-env
+      - run: echo insecure >> ~/.curlrc
       - run: yarn ${{ matrix.command }}
         env:
           AUTH_RUNNER_CLIENT_SECRET: ${{ secrets.AUTH_RUNNER_CLIENT_SECRET }}

diff --git a/.local.env b/.local.env
@@ -5,11 +5,11 @@ API_CORE_BASE=https://cube-creator.lndo.site/
 UI_BASE=https://app.cube-creator.lndo.site/
 
 # SPARQL endpoints
-STORE_QUERY_ENDPOINT=http://db.cube-creator.lndo.site/cube-creator/query
-STORE_UPDATE_ENDPOINT=http://db.cube-creator.lndo.site/cube-creator/update
-STORE_GRAPH_ENDPOINT=http://db.cube-creator.lndo.site/cube-creator/data
+STORE_QUERY_ENDPOINT=https://db.cube-creator.lndo.site/cube-creator/query
+STORE_UPDATE_ENDPOINT=https://db.cube-creator.lndo.site/cube-creator/update
+STORE_GRAPH_ENDPOINT=https://db.cube-creator.lndo.site/cube-creator/data
 STORE_ENGINE=fuseki
-PUBLIC_QUERY_ENDPOINT=http://db.cube-creator.lndo.site/shared-dimensions
+PUBLIC_QUERY_ENDPOINT=https://db.cube-creator.lndo.site/shared-dimensions
 TRIFID_UI=https://lindas.admin.ch/sparql
 VISUALIZE_UI=https://visualize.admin.ch
 
@@ -43,13 +43,15 @@ OTEL_LOG_LEVEL=debug
 MANAGED_DIMENSIONS_GRAPH=https://lindas.admin.ch/cube/dimension
 MANAGED_DIMENSIONS_API_BASE=https://cube-creator.lndo.site/
 MANAGED_DIMENSIONS_BASE=https://ld.admin.ch/cube/
-MANAGED_DIMENSIONS_STORE_QUERY_ENDPOINT=http://db.cube-creator.lndo.site/shared-dimensions/query
-MANAGED_DIMENSIONS_STORE_UPDATE_ENDPOINT=http://db.cube-creator.lndo.site/shared-dimensions/update
-MANAGED_DIMENSIONS_STORE_GRAPH_ENDPOINT=http://db.cube-creator.lndo.site/shared-dimensions/data
+MANAGED_DIMENSIONS_STORE_QUERY_ENDPOINT=https://db.cube-creator.lndo.site/shared-dimensions/query
+MANAGED_DIMENSIONS_STORE_UPDATE_ENDPOINT=https://db.cube-creator.lndo.site/shared-dimensions/update
+MANAGED_DIMENSIONS_STORE_GRAPH_ENDPOINT=https://db.cube-creator.lndo.site/shared-dimensions/data
 MANAGED_DIMENSIONS_STORE_ENGINE=fuseki
 
 # PX Cube sample
 PX_CUBE_BASE=https://environment.ld.admin.ch/foen/example/px-cube/
 PX_CUBE_GRAPH=http://example.org/px-cube
-PX_CUBE_QUERY_ENDPOINT=http://db.cube-creator.lndo.site/px-cube/query
-PX_CUBE_GRAPH_ENDPOINT=http://db.cube-creator.lndo.site/px-cube/data
+PX_CUBE_QUERY_ENDPOINT=https://db.cube-creator.lndo.site/px-cube/query
+PX_CUBE_GRAPH_ENDPOINT=https://db.cube-creator.lndo.site/px-cube/data
+
+NODE_TLS_REJECT_UNAUTHORIZED=0
diff --git a/.nvmrc b/.nvmrc
@@ -0,0 +1 @@
+18
diff --git a/README.md b/README.md
@@ -84,7 +84,7 @@ There are two types of e2e tests:
 
 ### API e2e tests
 
-Running the E2E tests can be done using: `docker-compose run --rm e2e-tests`, and `docker-compose run --rm e2e-tests -- --grep pattern` lets you select which tests to run.
+Running the E2E tests can be done using: `docker compose run --rm e2e-tests`, and `docker compose run --rm e2e-tests -- --grep pattern` lets you select which tests to run.
 
 For brevity, use npm script `npm run test:e2e --grep pattern`
 

diff --git a/apis/core/bootstrap/shapes/dataset.ts b/apis/core/bootstrap/shapes/dataset.ts
@@ -5,14 +5,19 @@ import { sparql, turtle } from '@tpluscode/rdf-string'
 import $rdf from 'rdf-ext'
 import { Draft } from '@cube-creator/model/Cube'
 import env from '@cube-creator/core/env'
-import { lindasQuery } from '../lib/query'
+import namespace from '@rdfjs/namespace'
+import { lindasQuery, lindasQueryTemplate } from '../lib/query'
+
+const euvoc = namespace('http://publications.europa.eu/ontology/euvoc#')
 
 const shapeId = shape('dataset/edit-metadata')
 const temporalFromTo = $rdf.namedNode(shapeId.value + '#temporalFromTo')
 const vcardOrganization = $rdf.namedNode(shapeId.value + '#vcardOrganization')
+const opendataSwissResources = $rdf.namedNode(shapeId.value + '#opendataSwissResources')
 
 const mainGroup = $rdf.blankNode()
 const opendataGroup = $rdf.blankNode()
+const opendataResourcesGroup = $rdf.blankNode()
 const aboutGroup = $rdf.blankNode()
 
 const themesQuery = sparql`construct {
@@ -46,6 +51,40 @@ const euThemesQuery = sparql`construct {
   }
 }`
 
+const fileTypesQuery = sparql`construct {
+  ?c a ${hydra.Collection} .
+  ?c ${hydra.member} ?term .
+  ?term ${rdfs.label} ?name .
+} WHERE {
+  BIND ( bnode('collection') as ?c )
+
+  graph <https://lindas.admin.ch/ontologies> {
+    ?term a ${euvoc.FileType} ;
+     ${skos.prefLabel} ?name ;
+      ${skos.inScheme} <http://publications.europa.eu/resource/authority/file-type> .
+
+    FILTER(regex(?name, "_q_", "i"))
+  }
+}`
+
+const mediaTypesQuery = sparql`construct {
+  ?c a ${hydra.Collection} .
+  ?c ${hydra.member} ?term .
+  ?term ${rdfs.label} ?name .
+} WHERE {
+  BIND ( bnode('collection') as ?c )
+
+  graph <https://lindas.admin.ch/ontologies> {
+    ?term a ${dcterms.MediaType} ;
+      ${dcterms.identifier} ?imt ;
+      ${skos.inScheme} <https://www.iana.org/assignments/media-types> .
+
+    BIND(str(?imt) as ?name)
+
+    FILTER(regex(?name, "_q_", "i"))
+  }
+}`
+
 const aboutQuery = sparql`construct {
   ?c a ${hydra.Collection} .
   ?c ${hydra.member} ?about .
@@ -118,7 +157,8 @@ export const DatasetShape = turtle`
 ${shapeId} {
   ${mainGroup} ${rdfs.label} "Cube Metadata" ; ${sh.order} 1 .
   ${opendataGroup} ${rdfs.label} "Opendata.swiss" ; ${sh.order} 2 .
-  ${aboutGroup} ${rdfs.label} "About" ; ${sh.order} 3 .
+  ${opendataResourcesGroup} ${rdfs.label} "Opendata.swiss Resources" ; ${sh.order} 3 .
+  ${aboutGroup} ${rdfs.label} "About" ; ${sh.order} 4 .
 
   ${shapeId} a ${sh.NodeShape}, ${hydra.Resource} ;
     ${sh.targetClass} ${_void.Dataset} ;
@@ -317,7 +357,7 @@ ${shapeId} {
       ${sh.order} 105 ;
       ${sh.description} "The Data Theme Category for the classification in the European Data Catalogs.";
       ${sh.nodeKind} ${sh.IRI} ;
-      ${sh.class} <http://publications.europa.eu/ontology/euvoc#DataTheme> ;
+      ${sh.class} ${euvoc.DataTheme} ;
       ${hydra.collection} ${lindasQuery(euThemesQuery)} ;
       ${sh.group} ${aboutGroup} ;
     ] ;
@@ -384,6 +424,83 @@ ${shapeId} {
       ${sh.message} "Metadata must not have the dcterms:identifier property. Please remove it in advanced RDF editor" ;
       ${sh.group} ${mainGroup} ;
     ] ;
+    ${sh.property} [
+      ${sh.name} "Resources" ;
+      ${sh.path} ${dcat.distribution} ;
+      ${sh.class} ${dcat.Distribution} ;
+      ${sh.nodeKind} ${sh.BlankNode} ;
+      ${dash.editor} ${dash.DetailsEditor} ;
+      ${sh.node} ${opendataSwissResources} ;
+      ${sh.order} 10 ;
+      ${sh.group} ${opendataResourcesGroup} ;
+    ] ;
+  .
+
+  ${opendataSwissResources} a ${sh.NodeShape} ;
+    ${sh.property} [
+      ${sh.name} "Access URL" ;
+      ${sh.path} ${dcat.accessURL} ;
+      ${sh.minCount} 1 ;
+      ${sh.nodeKind} ${sh.IRI} ;
+      ${sh.order} 10 ;
+      ${sh.message} "Web page address to access the data is required" ;
+    ] , [
+      ${sh.name} "Download URL" ;
+      ${sh.path} ${dcat.downloadURL} ;
+      ${sh.nodeKind} ${sh.IRI} ;
+      ${sh.order} 20 ;
+    ] , [
+      ${sh.name} "Title" ;
+      ${sh.path} ${dcterms.title} ;
+      ${sh.maxCount} 1 ;
+      ${sh.order} 30 ;
+      ${sh.datatype} ${rdf.langString} ;
+      ${sh.languageIn} ( ${supportedLanguages} ) ;
+    ] , [
+      ${sh.name} "Description" ;
+      ${sh.path} ${dcterms.description} ;
+      ${sh.order} 40 ;
+      ${sh.datatype} ${rdf.langString} ;
+      ${sh.languageIn} ( ${supportedLanguages} ) ;
+    ] , [
+      ${sh.name} "Media type" ;
+      ${sh.path} ${dcat.mediaType} ;
+      ${sh.maxCount} 1 ;
+      ${sh.nodeKind} ${sh.IRI} ;
+      ${dash.editor} ${dash.AutoCompleteEditor} ;
+      ${hydra.search} [
+        a ${hydra.IriTemplate} ;
+        ${hydra.template} "${lindasQueryTemplate(mediaTypesQuery, 'q')}" ; ;
+        ${hydra.mapping} [
+          ${hydra.property} ${hydra.freetextQuery} ;
+          ${hydra.variable} "q" ;
+          ${sh.minLength} 3 ;
+        ] ;
+      ] ;
+      ${sh.order} 50 ;
+    ] , [
+      ${sh.name} "Format" ;
+      ${sh.path} ${dcterms.format} ;
+      ${sh.maxCount} 1 ;
+      ${sh.nodeKind} ${sh.IRI} ;
+      ${dash.editor} ${dash.AutoCompleteEditor} ;
+      ${hydra.search} [
+        a ${hydra.IriTemplate} ;
+        ${hydra.template} "${lindasQueryTemplate(fileTypesQuery, 'q')}" ; ;
+        ${hydra.mapping} [
+          ${hydra.property} ${hydra.freetextQuery} ;
+          ${hydra.variable} "q" ;
+          ${sh.minLength} 3 ;
+        ] ;
+      ] ;
+      ${sh.order} 60 ;
+    ] , [
+      ${sh.path} ${dcat.byteSize} ;
+      ${sh.name} "Size [bytes]" ;
+      ${sh.maxCount} 1 ;
+      ${sh.datatype} ${xsd.decimal} ;
+      ${sh.order} 70 ;
+    ] ;
   .
 
   ${temporalFromTo} a ${sh.NodeShape} ;

diff --git a/apis/core/package.json b/apis/core/package.json
@@ -26,7 +26,7 @@
     "@sentry/node": "^6.2.0",
     "@sentry/tracing": "^6.2.0",
     "@tpluscode/rdf-ns-builders": "^1.0.0",
-    "@tpluscode/rdf-string": "^0.2.28",
+    "@tpluscode/rdf-string": "^0.2.29",
     "@tpluscode/rdfine": "^0.5.43",
     "@tpluscode/sparql-builder": "^0.3.24",
     "@uppy/companion": "^4.1.1",

diff --git a/cli/.test.env b/cli/.test.env
@@ -5,13 +5,13 @@ AUTH_RUNNER_CLIENT_ID=cube-creator-runner
 AUTH_RUNNER_ISSUER=https://keycloak.zazukoians.org/realms/zazuko-dev
 AUTH_RUNNER_CLIENT_SECRET=private
 
-GRAPH_QUERY_ENDPOINT=http://db.cube-creator.lndo.site/cube-creator/query
-GRAPH_STORE_ENDPOINT=http://db.cube-creator.lndo.site/cube-creator/data
+GRAPH_QUERY_ENDPOINT=https://db.cube-creator.lndo.site/cube-creator/query
+GRAPH_STORE_ENDPOINT=https://db.cube-creator.lndo.site/cube-creator/data
 GRAPH_STORE_USER=admin
 GRAPH_STORE_PASSWORD=password
 
-PUBLISH_GRAPH_STORE_ENDPOINT=http://db.cube-creator.lndo.site/cube-creator/data
-PUBLISH_GRAPH_QUERY_ENDPOINT=http://db.cube-creator.lndo.site/cube-creator
+PUBLISH_GRAPH_STORE_ENDPOINT=https://db.cube-creator.lndo.site/cube-creator/data
+PUBLISH_GRAPH_QUERY_ENDPOINT=https://db.cube-creator.lndo.site/cube-creator
 PUBLISH_GRAPH_STORE_USER=admin
 PUBLISH_GRAPH_STORE_PASSWORD=password
 

diff --git a/cli/README.md b/cli/README.md
@@ -96,12 +96,12 @@ Here's an example of converting local files using a locally-built image:
 1. (optionally) Ensure a fresh container is built
 
     ```
-    docker-composer build cli
+    docker compose build cli
     ```
-1. Run with docker-compose
+1. Run with `docker compose`
 
     ```
-    docker-compose run --rm cli transform \ 
+    docker compose run --rm cli transform \ 
         --to filesystem \
         --job <URI>
         --debug

diff --git a/e2e-tests/cube-projects/create-invalid.hydra b/e2e-tests/cube-projects/create-invalid.hydra
@@ -122,7 +122,7 @@ With Class api:EntryPoint {
                 <> cc:projectSourceKind <https://cube-creator.zazuko.com/vocab#projectSourceKind/ExistingCube> ;
                    rdfs:label "Test Project" ;
                    CubeProject:sourceCube <https://environment.ld.admin.ch/foen/ubd/28> ;
-                   CubeProject:sourceEndpoint <http://db.cube-creator.lndo.site/px-cube> ;
+                   CubeProject:sourceEndpoint <https://db.cube-creator.lndo.site/px-cube> ;
                    schema:maintainer <https://cube-creator.lndo.site/organization/bafu> .
                 ```
             } => {
@@ -142,7 +142,7 @@ With Class api:EntryPoint {
                 <> cc:projectSourceKind <https://cube-creator.zazuko.com/vocab#projectSourceKind/ExistingCube> ;
                    rdfs:label "Test Project" ;
                    CubeProject:sourceCube <https://environment.ld.admin.ch/foen/example/px-cube> ;
-                   CubeProject:sourceEndpoint <http://db.cube-creator.lndo.site/px-cube> ;
+                   CubeProject:sourceEndpoint <https://db.cube-creator.lndo.site/px-cube> ;
                    schema:maintainer <https://cube-creator.lndo.site/organization/bafu> .
                 ```
             } => {
@@ -162,7 +162,7 @@ With Class api:EntryPoint {
                 <> cc:projectSourceKind <https://cube-creator.zazuko.com/vocab#projectSourceKind/ExistingCube> ;
                    rdfs:label "Test Project" ;
                    CubeProject:sourceCube <https://environment.ld.admin.ch/bafu/example/px2> ;
-                   CubeProject:sourceEndpoint <http://db.cube-creator.lndo.site/px-cube> ;
+                   CubeProject:sourceEndpoint <https://db.cube-creator.lndo.site/px-cube> ;
                    schema:maintainer <https://cube-creator.lndo.site/organization/bafu> .
                 ```
             } => {

diff --git a/fuseki/sample-px.trig b/fuseki/sample-px.trig
@@ -31,7 +31,7 @@ graph <cube-project/px> {
     cc:projectSourceKind <https://cube-creator.zazuko.com/vocab#projectSourceKind/ExistingCube> ;
     CubeProject:sourceCube <https://environment.ld.admin.ch/foen/example/px-cube> ;
     CubeProject:sourceGraph <http://example.org/px-cube> ;
-    CubeProject:sourceEndpoint <http://db.cube-creator.lndo.site/px-cube> ;
+    CubeProject:sourceEndpoint <https://db.cube-creator.lndo.site/px-cube> ;
     cc:projectDetails <cube-project/px/details> ;
 }
 
@@ -135,6 +135,6 @@ graph <cube-project/px/jobs/import-job> {
     cc:dataset <cube-project/px/dataset> ;
     CubeProject:sourceCube <https://environment.ld.admin.ch/foen/example/px-cube> ;
     CubeProject:sourceGraph <http://example.org/px-cube> ;
-    CubeProject:sourceEndpoint <http://db.cube-creator.lndo.site/px-cube> ;
+    CubeProject:sourceEndpoint <https://db.cube-creator.lndo.site/px-cube> ;
   .
 }
diff --git a/package.json b/package.json
@@ -7,14 +7,14 @@
     "lint": "eslint . --ext .ts,.vue,.tsx --quiet --ignore-path .gitignore",
     "c8": "c8 --reporter lcovonly",
     "test": "yarn c8 -o coverage/apis mocha --recursive apis/**/*.test.ts packages/**/*.test.ts --grep @SPARQL --invert",
-    "test:queries": "yarn c8 -o coverage/queries mocha --recursive apis/**/*.test.ts ui/**/*.test.ts --grep @SPARQL",
+    "test:queries": "yarn c8 -o coverage/queries mocha -t 10000 --recursive apis/**/*.test.ts ui/**/*.test.ts --grep @SPARQL",
     "test:cli": "run-s test:cli:*",
     "test:cli:rest": "yarn c8 -o coverage/publish mocha --recursive cli/**/*.test.ts --grep @cube-creator/cli/lib/commands/* --invert",
     "test:cli:import": "yarn c8 -o coverage/publish mocha --recursive cli/**/*.test.ts --grep @cube-creator/cli/lib/commands/import",
     "test:cli:publish": "yarn c8 -o coverage/publish mocha --recursive cli/**/*.test.ts --grep @cube-creator/cli/lib/commands/publish",
     "test:cli:transform": "yarn c8 -o coverage/transform mocha --recursive cli/**/*.test.ts --grep @cube-creator/cli/lib/commands/transform",
     "test:cli:timeoutJobs": "yarn c8 -o coverage/transform mocha --recursive cli/**/*.test.ts --grep @cube-creator/cli/lib/commands/timeoutJobs",
-    "test:e2e": "docker-compose run --rm e2e-tests --",
+    "test:e2e": "docker compose run --rm e2e-tests --",
     "seed-data": "dotenv -e .local.env -- bash -c \"ts-node packages/testing/index.ts -i ubd dimensions px-cube hierarchies\""
   },
   "workspaces": [

diff --git a/packages/testing/lib/index.ts b/packages/testing/lib/index.ts
@@ -2,9 +2,9 @@ import StreamClient from 'sparql-http-client/StreamClient'
 import ParsingClient from 'sparql-http-client/ParsingClient'
 
 const endpoints = (db: 'cube-creator' | 'shared-dimensions') => ({
-  updateUrl: `http://db.cube-creator.lndo.site/${db}/update`,
-  endpointUrl: `http://db.cube-creator.lndo.site/${db}/query`,
-  storeUrl: `http://db.cube-creator.lndo.site/${db}/data`,
+  updateUrl: `https://db.cube-creator.lndo.site/${db}/update`,
+  endpointUrl: `https://db.cube-creator.lndo.site/${db}/query`,
+  storeUrl: `https://db.cube-creator.lndo.site/${db}/data`,
 })
 
 export const ccClients = {

diff --git a/ui/src/forms/editors/TextFieldWithLangEditor.vue b/ui/src/forms/editors/TextFieldWithLangEditor.vue
@@ -63,7 +63,7 @@ export default defineComponent({
 </script>
 
 <style scoped>
-.text-input {
+.field :first-child {
   flex-grow: 1;
 }
 </style>