Merge pull request #155 from zazuko/migrate-csvw

Migrate csvw

.github/workflows/ci.yaml

@@ -11,12 +11,13 @@ jobs:
           - '16'
         package:
           - barnard59
+          - barnard59-base
           - barnard59-core
+          - barnard59-csvw
+          - barnard59-ftp
+          - barnard59-graph-store
           - barnard59-rdf
           - barnard59-sparql
-          - barnard59-base
-          - barnard59-graph-store
-          - barnard59-ftp
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3

README.md

@@ -9,11 +9,12 @@ In this monorepo you will find the various `barnard59-*` packages:
 | Package                                         | Latest version                                                                                  |                                                                                        |
 |-------------------------------------------------|-------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------|
 | [`barnard59`](packages/cli)                     | [![](https://badge.fury.io/js/barnard59.svg)](https://npm.im/barnard59)                         | CLI to run pipelines                                                                   |
-| [`barnard59-core`](packages/core)               | [![](https://badge.fury.io/js/barnard59-core.svg)](https://npm.im/barnard59-core)               | Core package                                                                           |
 | [`barnard59-base`](packages/base)               | [![](https://badge.fury.io/js/barnard59-base.svg)](https://npm.im/barnard59-base)               | Provides the basic pipeline steps                                                      |
+| [`barnard59-core`](packages/core)               | [![](https://badge.fury.io/js/barnard59-core.svg)](https://npm.im/barnard59-core)               | Core package                                                                           |
+| [`barnard59-csvw`](packages/csvw)               | [![](https://badge.fury.io/js/barnard59-csvw.svg)](https://npm.im/barnard59-csvw)               | Simplifies handling CSVW mapping documents in pipelines                                |
 | [`barnard59-ftp`](packages/ftp)                 | [![](https://badge.fury.io/js/barnard59-ftp.svg)](https://npm.im/barnard59-ftp)                 | FTP support for Linked Data pipelines                                                  |
 | [`barnard59-graph-store`](packages/graph-store) | [![](https://badge.fury.io/js/barnard59-graph-store.svg)](https://npm.im/barnard59-graph-store) | [SPARQL Graph Store Protocol](https://www.w3.org/TR/sparql11-http-rdf-update/) support |
-| [`barnard59-sparql`](packages/sparql)           | [![](https://badge.fury.io/js/barnard59-sparql.svg)](https://npm.im/barnard59-sparql)           | Query SPARQL endpoint from pipeline                                                    |
 | [`barnard59-rdf`](packages/rdf)                 | [![](https://badge.fury.io/js/barnard59-rdf.svg)](https://npm.im/barnard59-rdf)                 | Operations for RDF/JS quads and datasets                                               |
+| [`barnard59-sparql`](packages/sparql)           | [![](https://badge.fury.io/js/barnard59-sparql.svg)](https://npm.im/barnard59-sparql)           | Query SPARQL endpoint from pipeline                                                    |
 
 More to come as we gradually consolidate other, initially separate repositories.

codecov.yml

@@ -3,8 +3,10 @@ flag_management:
     carryforward: true
   individual_flags:
     - name: barnard59
+    - name: barnard59-base
     - name: barnard59-core
+    - name: barnard59-csvw
+    - name: barnard59-ftp
+    - name: barnard59-graph-store
     - name: barnard59-rdf
     - name: barnard59-sparql
-    - name: barnard59-base
-    - name: barnard59-graph-store

packages/csvw/README.md

@@ -0,0 +1,35 @@
+#  bardnard59-csvw
+
+Simplifies handling CSVW mapping documents in barnard59 pipelines
+
+## Install
+
+```
+npm install barnard59-csvw --save
+```
+
+## Exported steps
+
+### `fetch`
+
+A step to automate loading CSVW mapping documents and the source CSV, by following the `csvw:url` property. The URL can be local filesystem path or remote URL.
+
+| Argument | Type | Description |
+| -- | -- | -- |
+| `csvw` | `string` | Local path or URL of the mapping to load |
+
+### Example: Loading CSVW from filesystem
+
+```turtle
+@prefix p: <https://pipeline.described.at/> .
+@prefix code: <https://code.described.at/> .
+
+<#CsvwStep> a p:Step;
+  code:implementedBy [ a code:EcmaScriptModule;
+    code:link <node:barnard59-csvw/fetch.js#default>
+  ];
+  code:arguments [
+    code:name "csvw";
+    code:value "file:/test/mappings/remote.csvw.json"
+  ].
+```

packages/csvw/fetch.js

@@ -0,0 +1,25 @@
+import rdf from '@zazuko/env'
+import toReadable from 'duplex-to/readable.js'
+import { PassThrough } from 'readable-stream'
+import fetchData from './lib/fetchData.js'
+import fetchMetadata from './lib/fetchMetadata.js'
+
+function fetch({ csvw }) {
+  const output = new PassThrough()
+
+  Promise.resolve().then(async () => {
+    try {
+      const metadata = await fetchMetadata(csvw)
+      const url = metadata.any().has(rdf.ns.csvw.url).out(rdf.ns.csvw.url)
+      const dataStream = await fetchData(url.value)
+
+      dataStream.pipe(output)
+    } catch (err) {
+      output.destroy(err)
+    }
+  })
+
+  return toReadable(output)
+}
+
+export default fetch

packages/csvw/index.js

@@ -0,0 +1,5 @@
+import fetch from './fetch.js'
+
+export {
+  fetch,
+}

packages/csvw/lib/checkResponse.js

@@ -0,0 +1,7 @@
+async function checkResponse(res) {
+  if (!res.ok) {
+    throw new Error(`${res.statusText}(${res.status}): ${await res.text()}`)
+  }
+}
+
+export default checkResponse

packages/csvw/lib/commonFetch.js

@@ -0,0 +1,11 @@
+import fileFetch from 'file-fetch'
+import httpFetch from 'node-fetch'
+import protoFetch from 'proto-fetch'
+
+const commonFetch = protoFetch({
+  file: fileFetch,
+  http: httpFetch,
+  https: httpFetch,
+})
+
+export default commonFetch

packages/csvw/lib/fetchData.js

@@ -0,0 +1,12 @@
+import checkResponse from './checkResponse.js'
+import commonFetch from './commonFetch.js'
+
+async function fetchData(url) {
+  const res = await commonFetch(url)
+
+  await checkResponse(res)
+
+  return res.body
+}
+
+export default fetchData

packages/csvw/lib/fetchMetadata.js

@@ -0,0 +1,22 @@
+import rdfFetch from '@rdfjs/fetch'
+import rdf from '@zazuko/env'
+import checkResponse from './checkResponse.js'
+import commonFetch from './commonFetch.js'
+
+async function fetchMetadata(url) {
+  const res = await rdfFetch(url.toString(), {
+    contentTypeLookup: extension => extension === '.json' ? 'application/ld+json' : undefined,
+    factory: rdf,
+    fetch: commonFetch,
+  })
+
+  await checkResponse(res)
+
+  if (!res.dataset) {
+    throw new Error('response is empty')
+  }
+
+  return rdf.clownface({ dataset: await res.dataset() })
+}
+
+export default fetchMetadata

packages/csvw/manifest.ttl

@@ -0,0 +1,11 @@
+@base <http://barnard59.zazuko.com/operations/csvw/> .
+@prefix code: <https://code.described.at/> .
+@prefix p: <https://pipeline.described.at/> .
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
+
+<fetch> a p:Operation, p:Readable;
+  rdfs:label "Fetch CSVW";
+  rdfs:comment "Loads a CSVW file from the local filesystem or the web depending on the given argument";
+  code:implementedBy [ a code:EcmaScriptModule;
+    code:link <node:barnard59-csvw/fetch.js#default>
+  ].

packages/csvw/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "barnard59-csvw",
+  "version": "1.0.0",
+  "description": "Simplifies handling CSVW mapping documents in barnard59 pipelines",
+  "type": "module",
+  "main": "index.js",
+  "scripts": {
+    "test": "mocha"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git://github.com/zazuko/barnard59.git",
+    "directory": "packages/csvw"
+  },
+  "keywords": [],
+  "author": "Zazuko GmbH",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/zazuko/barnard59/issues"
+  },
+  "homepage": "https://github.com/zazuko/barnard59",
+  "dependencies": {
+    "@rdfjs/fetch": "^2.1.0",
+    "@zazuko/env": "^1.0.1",
+    "duplex-to": "^1.0.1",
+    "file-fetch": "^1.7.0",
+    "node-fetch": "^3.0.0",
+    "proto-fetch": "^1.0.0",
+    "readable-stream": "^3.6.0"
+  },
+  "devDependencies": {
+    "express-as-promise": "^1.2.0",
+    "get-stream": "^7.0.1",
+    "is-stream": "^3.0.0"
+  },
+  "engines": {
+    "node": ">= 14.0.0"
+  }
+}