Merge pull request #173 from zazuko/loader

extract loader package

.changeset/brave-tips-speak.md

@@ -0,0 +1,5 @@
+---
+"@zazuko/vocabulary-loader": major
+---
+
+First release

.changeset/empty-lions-roll.md

@@ -0,0 +1,5 @@
+---
+"@zazuko/prefixes": patch
+---
+
+Export required type

packages/loader/index.ts

@@ -0,0 +1,90 @@
+/* eslint-disable no-console */
+import { Stream, DataFactory, DatasetCore, DatasetCoreFactory } from 'rdf-js'
+import { Readable } from 'stream'
+import rdf from '@zazuko/env'
+import type { Environment } from '@rdfjs/environment/Environment.js'
+import ParserN3 from '@rdfjs/parser-n3'
+import fromStream from 'rdf-dataset-ext/fromStream.js'
+import addAll from 'rdf-dataset-ext/addAll.js'
+import toStream from 'rdf-dataset-ext/toStream.js'
+import { loadDatasetStream } from './lib/loadDataset.js'
+
+export type Datasets<P extends Record<string, string>> = Record<keyof P, DatasetCore>
+
+export interface VocabulariesOptions<P extends Record<string, string>> {
+  only?: (keyof P)[] | null
+  factory?: Environment<DatasetCoreFactory | DataFactory>
+}
+
+export interface VocabulariesDatasetOptions<P extends Record<string, string>> extends VocabulariesOptions<P> {
+  stream?: false
+}
+
+export interface VocabulariesStreamOptions<P extends Record<string, string>> extends VocabulariesOptions<P> {
+  stream: true
+}
+
+export interface Loader<P extends Record<string, string>> {
+  (options?: VocabulariesDatasetOptions<P>): Promise<Datasets<P>>
+  (options: VocabulariesStreamOptions<P>): Promise<Stream & Readable>
+}
+
+export function create<P extends Record<string, string>>(prefixMap: P): Loader<P> {
+  return async function vocabularies({ only = null, factory = rdf, stream = false }: VocabulariesDatasetOptions<P> | VocabulariesStreamOptions<P> = {}) {
+    let selectedPrefixes: (keyof typeof prefixMap)[] = []
+
+    if (!!only && Array.isArray(only)) {
+      only.forEach((prefix: keyof P) => {
+        if (prefix in prefixMap) {
+          selectedPrefixes.push(prefix)
+        } else {
+          console.warn(`unknown prefix '${String(prefix)}'`)
+        }
+      })
+    }
+    if (!selectedPrefixes.length) {
+      selectedPrefixes = Object.keys(prefixMap)
+    }
+
+    const promises = selectedPrefixes.map((prefix) => loadFile(prefix, { customSelection: !!only, factory }))
+    const datasets = await Promise.all(promises)
+
+    if (stream !== false) {
+      let combinedDataset = factory.dataset()
+      datasets.forEach((dataset) => {
+        if (dataset && dataset.size) {
+          combinedDataset = addAll(combinedDataset, dataset)
+        }
+      })
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      return toStream(combinedDataset) as any
+    }
+
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const result: Datasets<P> = {} as any
+    datasets.forEach((dataset, i) => {
+      if (dataset && dataset.size) {
+        result[selectedPrefixes[i]] = dataset
+      }
+    })
+    return result
+  }
+}
+
+interface LoadFileOptions {
+  customSelection?: boolean
+  factory: Environment<DatasetCoreFactory>
+}
+
+export async function loadFile<P extends Record<string, string>>(prefix: keyof P, { customSelection, factory }: LoadFileOptions) {
+  try {
+    const parserN3 = new ParserN3()
+    const readStream = await loadDatasetStream(String(prefix))
+    const quadStream = parserN3.import(readStream)
+    return fromStream(factory.dataset(), quadStream)
+  } catch {
+    if (customSelection) {
+      console.warn(`unavailable prefix '${String(prefix)}'`)
+    }
+  }
+}

packages/vocabularies/lib/loadDataset.ts → packages/loader/lib/loadDataset.ts

@@ -1,6 +1,6 @@
 import fs from 'fs'
 import module from 'module'
-import type prefixes from '../prefixes.js'
+import type prefixes from '@zazuko/prefixes'
 
 const { resolve } = module.createRequire(import.meta.url)
 

packages/loader/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "@zazuko/vocabulary-loader",
+  "version": "0.0.0",
+  "type": "module",
+  "main": "index.js",
+  "files": [
+    "**/*.js",
+    "**/*.d.ts"
+  ],
+  "dependencies": {
+    "@zazuko/env": "^1.6.1",
+    "@rdfjs/parser-n3": "^2.0.1",
+    "@types/rdfjs__environment": "^0.1",
+    "rdf-dataset-ext": "^1.1.0"
+  }
+}

packages/prefixes/.build/prefixes.sh

@@ -20,7 +20,7 @@ const prefixes = {
 export interface Prefixes extends Readonly<typeof prefixes> {
 
 }
-interface CustomPrefixes {
+export interface CustomPrefixes {
   [key: string]: string
 }
 

packages/prefixes/prefixes.ts

@@ -114,7 +114,7 @@ const prefixes = {
 export interface Prefixes extends Readonly<typeof prefixes> {
 
 }
-interface CustomPrefixes {
+export interface CustomPrefixes {
   [key: string]: string
 }
 

packages/vocabularies/bin/vocab.js

@@ -3,7 +3,7 @@
 import { promisify } from 'util'
 import { program } from 'commander'
 import stream from 'readable-stream'
-import { loadDatasetStream } from '../lib/loadDataset.js'
+import { loadDatasetStream } from '@zazuko/vocabulary-loader/lib/loadDataset.js'
 import prefixes from '../prefixes.js'
 
 const finished = promisify(stream.finished)

packages/vocabularies/expandWithCheck.ts

@@ -4,7 +4,8 @@ import { expand, getParts } from './expand.js'
 import { vocabularies, Datasets } from './vocabularies.js'
 
 // memoizing the prefixes already used in 'expand'
-export const loadedPrefixes: Datasets = {}
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export const loadedPrefixes: Datasets<any> = {}
 
 type Types = (string | NamedNode)[]
 

packages/vocabularies/package.json

@@ -20,7 +20,7 @@
   },
   "dependencies": {
     "@rdfjs/parser-n3": "^2.0.1",
-    "@types/rdfjs__environment": "^0.1",
+    "@zazuko/vocabulary-loader": "^0.0.0",
     "@vocabulary/acl": "^1.0.0",
     "@vocabulary/as": "^1.0.0",
     "@vocabulary/bibo": "^1.0.0",

packages/vocabularies/vocabularies.ts

@@ -1,84 +1,6 @@
-/* eslint-disable no-console */
-import { Stream, DataFactory, DatasetCore, DatasetCoreFactory } from 'rdf-js'
-import { Readable } from 'stream'
-import rdf from '@zazuko/env'
-import type { Environment } from '@rdfjs/environment/Environment.js'
-import prefixes from '@zazuko/prefixes'
-import ParserN3 from '@rdfjs/parser-n3'
-import fromStream from 'rdf-dataset-ext/fromStream.js'
-import addAll from 'rdf-dataset-ext/addAll.js'
-import toStream from 'rdf-dataset-ext/toStream.js'
-import { loadDatasetStream } from './lib/loadDataset.js'
+import { create } from '@zazuko/vocabulary-loader'
+import prefixes from './prefixes.js'
 
-export type Datasets = Record<keyof typeof prefixes, DatasetCore>
+export { loadFile, Datasets } from '@zazuko/vocabulary-loader'
 
-interface VocabulariesOptions {
-  only?: (keyof typeof prefixes)[] | null
-  factory?: Environment<DatasetCoreFactory | DataFactory>
-}
-
-interface VocabulariesDatasetOptions extends VocabulariesOptions {
-  stream?: false
-}
-
-interface VocabulariesStreamOptions extends VocabulariesOptions {
-  stream: true
-}
-
-export async function vocabularies (options?: VocabulariesDatasetOptions): Promise<Datasets>
-export async function vocabularies (options: VocabulariesStreamOptions): Promise<Stream & Readable>
-export async function vocabularies({ only = null, factory = rdf, stream = false }: VocabulariesDatasetOptions | VocabulariesStreamOptions = {}) {
-  let selectedPrefixes: (keyof typeof prefixes)[] = []
-
-  if (!!only && Array.isArray(only)) {
-    only.forEach((prefix: keyof typeof prefixes) => {
-      if (prefix in prefixes) {
-        selectedPrefixes.push(prefix)
-      } else {
-        console.warn(`unknown prefix '${prefix}'`)
-      }
-    })
-  }
-  if (!selectedPrefixes.length) {
-    selectedPrefixes = Object.keys(prefixes)
-  }
-
-  const promises = selectedPrefixes.map((prefix) => loadFile(prefix, { customSelection: !!only, factory }))
-  const datasets = await Promise.all(promises)
-
-  if (stream !== false) {
-    let combinedDataset = factory.dataset()
-    datasets.forEach((dataset) => {
-      if (dataset && dataset.size) {
-        combinedDataset = addAll(combinedDataset, dataset)
-      }
-    })
-    return toStream(combinedDataset) as any
-  }
-
-  const result: Datasets = {}
-  datasets.forEach((dataset, i) => {
-    if (dataset && dataset.size) {
-      result[selectedPrefixes[i]] = dataset
-    }
-  })
-  return result
-}
-
-interface LoadFileOptions {
-  customSelection?: boolean
-  factory: Environment<DatasetCoreFactory>
-}
-
-export async function loadFile(prefix: keyof typeof prefixes, { customSelection, factory }: LoadFileOptions) {
-  try {
-    const parserN3 = new ParserN3()
-    const readStream = await loadDatasetStream(prefix)
-    const quadStream = parserN3.import(readStream)
-    return fromStream(factory.dataset(), quadStream)
-  } catch {
-    if (customSelection) {
-      console.warn(`unavailable prefix '${prefix}'`)
-    }
-  }
-}
+export const vocabularies = create(prefixes)