Merge pull request #162 from zazuko/migrate-formats

Migrate formats package

.changeset/dull-bees-mix.md

@@ -0,0 +1,5 @@
+---
+"barnard59-formats": patch
+---
+
+The package would use `rdf-ext` but it was not a dependency. Using `@zazuko/env` instead

.github/workflows/ci.yaml

@@ -14,6 +14,7 @@ jobs:
           - barnard59-base
           - barnard59-core
           - barnard59-csvw
+          - barnard59-formats
           - barnard59-ftp
           - barnard59-graph-store
           - barnard59-rdf

README.md

@@ -12,6 +12,7 @@ In this monorepo you will find the various `barnard59-*` packages:
 | [`barnard59-base`](packages/base)               | [![](https://badge.fury.io/js/barnard59-base.svg)](https://npm.im/barnard59-base)               | Provides the basic pipeline steps                                                      |
 | [`barnard59-core`](packages/core)               | [![](https://badge.fury.io/js/barnard59-core.svg)](https://npm.im/barnard59-core)               | Core package                                                                           |
 | [`barnard59-csvw`](packages/csvw)               | [![](https://badge.fury.io/js/barnard59-csvw.svg)](https://npm.im/barnard59-csvw)               | Simplifies handling CSVW mapping documents in pipelines                                |
+| [`barnard59-formats`](packages/formats)         | [![](https://badge.fury.io/js/barnard59-formats.svg)](https://npm.im/barnard59-formats)         | Parse and serialize various formats in Linked Data pipelines                           |
 | [`barnard59-ftp`](packages/ftp)                 | [![](https://badge.fury.io/js/barnard59-ftp.svg)](https://npm.im/barnard59-ftp)                 | FTP support for Linked Data pipelines                                                  |
 | [`barnard59-graph-store`](packages/graph-store) | [![](https://badge.fury.io/js/barnard59-graph-store.svg)](https://npm.im/barnard59-graph-store) | [SPARQL Graph Store Protocol](https://www.w3.org/TR/sparql11-http-rdf-update/) support |
 | [`barnard59-rdf`](packages/rdf)                 | [![](https://badge.fury.io/js/barnard59-rdf.svg)](https://npm.im/barnard59-rdf)                 | Operations for RDF/JS quads and datasets                                               |

codecov.yml

@@ -6,6 +6,7 @@ flag_management:
     - name: barnard59-base
     - name: barnard59-core
     - name: barnard59-csvw
+    - name: barnard59-formats
     - name: barnard59-ftp
     - name: barnard59-graph-store
     - name: barnard59-rdf

packages/formats/CHANGELOG.md

@@ -0,0 +1,32 @@
+# v1.0.0
+
+## 1.4.0
+
+### Minor Changes
+
+- f09e8b0: Forward n3 step options to parser (closes #24). For example, to parse n3 rules
+
+  ```turtle
+  [
+    a :Step ;
+    code:implementedBy
+      [
+        a code:EcmaScriptModule ;
+        code:link <node:barnard59-formats/n3.js#parse>
+      ] ;
+    code:arguments
+      [
+        code:name "format" ;
+        code:value "text/n3" ;
+      ] ;
+  ]
+  ```
+
+## 1.3.1
+
+### Patch Changes
+
+- 59d713f: Updated RDF/JS packages to v2
+- 396d36e: Correct rdf/xml usage in manifest (closes #20)
+
+- Moved to JavaScript modules

packages/formats/README.md

@@ -0,0 +1,3 @@
+# barnard59-formats
+
+This package provides support for various formats like CSV on the Web, JSON-LD and N-Triples for Barnard59 Linked Data pipelines.

packages/formats/csvw.js

@@ -0,0 +1,50 @@
+import sinkToDuplex from '@rdfjs/sink-to-duplex'
+import CsvwParser from 'rdf-parser-csvw'
+import tracer from './lib/tracer.js'
+import { toDataset } from './lib/stream.js'
+
+function parse(args) {
+  let metadata
+  let relaxColumnCount = false
+  let skipLinesWithError = false
+  let timezone = 'local'
+
+  if (args.metadata) {
+    metadata = args.metadata
+
+    if (typeof args.relaxColumnCount !== 'undefined') {
+      relaxColumnCount = Boolean(args.relaxColumnCount)
+    }
+
+    if (typeof args.skipLinesWithError !== 'undefined') {
+      skipLinesWithError = Boolean(args.skipLinesWithError)
+    }
+
+    if (typeof args.timezone !== 'undefined') {
+      timezone = args.timezone
+    }
+  } else {
+    metadata = args
+  }
+
+  return tracer.startActiveSpan('csvw:parse', async span => {
+    try {
+      const dataset = await toDataset(metadata)
+      span.addEvent('metadata')
+      return sinkToDuplex(new CsvwParser({
+        metadata: dataset,
+        relaxColumnCount,
+        skipLinesWithError,
+        timezone,
+      }), {
+        readableObjectMode: true,
+      })
+    } finally {
+      span.end()
+    }
+  })
+}
+
+export {
+  parse,
+}

packages/formats/index.js

@@ -0,0 +1,5 @@
+export { parse as csvwParse } from './csvw.js'
+export { parse as jsonldParse, parseObject as jsonldParseObject, serialize as jsonldSerialize } from './jsonld.js'
+export { parse as n3Parse } from './n3.js'
+export { serialize as ntriplesSerialize } from './ntriples.js'
+export { parse as rdfxmlParse } from './rdf-xml.js'

packages/formats/jsonld.js

@@ -0,0 +1,51 @@
+import { SpanStatusCode } from '@opentelemetry/api'
+import Parser from '@rdfjs/parser-jsonld'
+import FsDocumentLoader from '@rdfjs/parser-jsonld/FsDocumentLoader.js'
+import Serializer from '@rdfjs/serializer-jsonld'
+import sinkToDuplex from '@rdfjs/sink-to-duplex'
+import { combine, jsonStringify } from 'barnard59-base'
+import tracer from './lib/tracer.js'
+
+function parse({ localContext } = {}) {
+  let documentLoader = null
+
+  if (localContext) {
+    if (typeof localContext === 'string') {
+      localContext = JSON.parse(localContext)
+    }
+
+    documentLoader = new FsDocumentLoader(localContext)
+  }
+
+  return tracer.startActiveSpan('jsonld:parse', span => {
+    const stream = sinkToDuplex(new Parser({ documentLoader }), { objectMode: true })
+    stream.on('error', err => {
+      span.recordException(err)
+      span.setStatus({ code: SpanStatusCode.ERROR, message: err.message })
+      span.end()
+    })
+    stream.on('finish', () => span.end())
+    return stream
+  })
+}
+
+const parseObject = () => {
+  return tracer.startActiveSpan('jsonld:parse.object', span => {
+    const stream = combine([jsonStringify(), parse()], { objectMode: true })
+    stream.on('error', err => {
+      span.recordException(err)
+      span.setStatus({ code: SpanStatusCode.ERROR, message: err.message })
+      span.end()
+    })
+    stream.on('finish', () => span.end())
+    return stream
+  })
+}
+
+parse.object = parseObject
+
+function serialize() {
+  return sinkToDuplex(new Serializer(), { objectMode: true })
+}
+
+export { parse, parseObject, serialize }

packages/formats/lib/stream.js

@@ -0,0 +1,10 @@
+import fromStream from 'rdf-dataset-ext/fromStream.js'
+import rdf from '@zazuko/env'
+
+export function toDataset(streamOrDataset) {
+  if (!streamOrDataset.readable) {
+    return Promise.resolve(streamOrDataset)
+  }
+
+  return fromStream(rdf.dataset(), streamOrDataset)
+}

packages/formats/lib/tracer.js

@@ -0,0 +1,6 @@
+import { trace } from '@opentelemetry/api'
+
+// TODO: set library version here
+const tracer = trace.getTracer('barnard59-formats')
+
+export default tracer

packages/formats/manifest.ttl

@@ -0,0 +1,60 @@
+@base <http://barnard59.zazuko.com/operations/formats/> .
+@prefix code: <https://code.described.at/> .
+@prefix p: <https://pipeline.described.at/> .
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
+
+<csvw/parse> a p:Operation, p:Writable, p:ReadableObjectMode;
+  rdfs:label "Parse CSV on the Web";
+  rdfs:comment "Parses the given CSV stream using the given metadata.";
+  code:implementedBy [ a code:EcmaScript;
+    code:link <node:barnard59-formats/csvw.js#parse>
+  ].
+
+<jsonld/parse> a p:Operation, p:Writable, p:ReadableObjectMode;
+  rdfs:label "Parse JSON-LD";
+  rdfs:comment "Parses the given JSON-LD stream.";
+  code:implementedBy [ a code:EcmaScript;
+    code:link <node:barnard59-formats/jsonld.js#parse>
+  ].
+
+<jsonld/parse/object> a p:Operation, p:WritableObjectMode, p:ReadableObjectMode;
+  rdfs:label "Parse JSON-LD (Object)";
+  rdfs:comment "Parses the given JSON-LD stream.";
+  code:implementedBy [ a code:EcmaScript;
+    code:link <node:barnard59-formats/jsonld.js#parseObject>
+  ].
+
+<jsonld/serialize> a p:Operation, p:WritableObjectMode, p:Readable;
+  rdfs:label "Serialize JSON-LD";
+  rdfs:comment "Serializes the given RDF/JS Quads to JSON-LD.";
+  code:implementedBy [ a code:EcmaScript;
+    code:link <node:barnard59-formats/jsonld.js#serialize>
+  ].
+
+<n3/parse> a p:Operation, p:Writable, p:ReadableObjectMode;
+  rdfs:label "Parse N3";
+  rdfs:comment "Parses the given N3 stream.";
+  code:implementedBy [ a code:EcmaScript;
+    code:link <node:barnard59-formats/n3.js#parse>
+  ].
+
+<ntriples/serialize> a p:Operation, p:WritableObjectMode, p:Readable;
+  rdfs:label "Serialize N-Triples";
+  rdfs:comment "Serializes the given RDF/JS Quads to N-Triples.";
+  code:implementedBy [ a code:EcmaScript;
+    code:link <node:barnard59-formats/ntriples.js#serialize>
+  ].
+
+<rdf-xml/parse> a p:Operation, p:Writable, p:ReadableObjectMode;
+  rdfs:label "Parse RDF/XML";
+  rdfs:comment "Parses the given RDF/XML stream.";
+  code:implementedBy [ a code:EcmaScript;
+    code:link <node:barnard59-formats/rdf-xml.js#parse>
+  ].
+
+<xlsx/parse> a p:Operation, p:Writable, p:ReadableObjectMode;
+  rdfs:label "Parse XSLX files based on the CSV on the Web standard";
+  rdfs:comment "Parses the given XLSX stream using the given metadata.";
+  code:implementedBy [ a code:EcmaScript;
+    code:link <node:barnard59-formats/xlsx.js#parse>
+  ].

packages/formats/n3.js

@@ -0,0 +1,11 @@
+import Parser from '@rdfjs/parser-n3'
+import sinkToDuplex from '@rdfjs/sink-to-duplex'
+
+function parse(args) {
+  return sinkToDuplex(new Parser(args), {
+    readableObjectMode: true,
+    writableObjectMode: true,
+  })
+}
+
+export { parse }

packages/formats/ntriples.js

@@ -0,0 +1,11 @@
+import Serializer from '@rdfjs/serializer-ntriples'
+import sinkToDuplex from '@rdfjs/sink-to-duplex'
+
+function serialize() {
+  return sinkToDuplex(new Serializer(), {
+    readableObjectMode: true,
+    writableObjectMode: true,
+  })
+}
+
+export { serialize }