From 7d64ad522697d3387a2c79810f1da7a301ad8c93 Mon Sep 17 00:00:00 2001 From: Felipe Lalanne Date: Fri, 27 Dec 2024 15:37:53 -0300 Subject: [PATCH 1/2] Add class to work with a Universe of contracts The helper class includes a `fromFs` function, that allows to build a universe of contracts searching for JSON files under a directory. Change-type: minor --- lib/index.ts | 2 + lib/universe.ts | 131 ++++++++++++++++++ .../contracts/arch.sw/aarch64/contract.json | 15 ++ .../contracts/arch.sw/amd64/contract.json | 15 ++ .../contracts/arch.sw/armv7hf/contract.json | 15 ++ .../contracts/arch.sw/rpi/contract.json | 15 ++ .../hw.device-type/intel-nuc/contract.json | 29 ++++ .../hw.device-type/jetson-nano/contract.json | 24 ++++ .../hw.device-type/raspberry-pi/contract.json | 29 ++++ .../raspberry-pi2/contract.json | 29 ++++ .../hw.device-type/raspberrypi3/contract.json | 27 ++++ .../raspberrypi4-64/contract.json | 27 ++++ tests/universe/from-fs.spec.ts | 130 +++++++++++++++++ 13 files changed, 488 insertions(+) create mode 100644 lib/universe.ts create mode 100644 tests/universe/contracts/arch.sw/aarch64/contract.json create mode 100644 tests/universe/contracts/arch.sw/amd64/contract.json create mode 100644 tests/universe/contracts/arch.sw/armv7hf/contract.json create mode 100644 tests/universe/contracts/arch.sw/rpi/contract.json create mode 100644 tests/universe/contracts/hw.device-type/intel-nuc/contract.json create mode 100644 tests/universe/contracts/hw.device-type/jetson-nano/contract.json create mode 100644 tests/universe/contracts/hw.device-type/raspberry-pi/contract.json create mode 100644 tests/universe/contracts/hw.device-type/raspberry-pi2/contract.json create mode 100644 tests/universe/contracts/hw.device-type/raspberrypi3/contract.json create mode 100644 tests/universe/contracts/hw.device-type/raspberrypi4-64/contract.json create mode 100644 tests/universe/from-fs.spec.ts diff --git a/lib/index.ts b/lib/index.ts index e6bebb9..9c7c7dd 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -16,6 +16,7 @@ import { } from './types/types'; import Contract from './contract'; import Blueprint from './blueprint'; +import Universe from './universe'; import { buildTemplate } from './partials'; import { parse as parseCardinality } from './cardinality'; @@ -25,6 +26,7 @@ export { BlueprintObject, Contract, Blueprint, + Universe, buildTemplate, parseCardinality, }; diff --git a/lib/universe.ts b/lib/universe.ts new file mode 100644 index 0000000..43aeddb --- /dev/null +++ b/lib/universe.ts @@ -0,0 +1,131 @@ +import path from 'path'; +import { promises as fs } from 'fs'; +import type { Stats } from 'fs'; + +import Contract from './contract'; +import { UNIVERSE } from './types/types'; + +/** + * @summary Run the callback function concurrently on elements from the given iterator + * @function + * @memberof module:universe + * + * Gets up to `concurrency` elements from the given iterator and apply the asynchronous function + * concurrently using `Promise.all`. + * + * If at any point the call to the callback fails, the function will throw the error + * + * @param it - iterator of elements to go traverse + * @param callbackFn - a function to execute for each element in the iterator + * @param concurrency - number of elements to apply the function at the same time. Default to 1 + */ +export async function concurrentForEach( + it: IterableIterator, + callbackFn: (t: T) => PromiseLike, + concurrency = 1, +): Promise { + const run = async () => { + const next = it.next(); + if (next.value && !next.done) { + await callbackFn(next.value); + await run(); + } + }; + const runs = []; + for (let i = 0; i < concurrency; i++) { + runs.push(run()); + } + await Promise.all(runs); +} + +/** + * @summary recursively find all files under the directory that match the given filter + * @function + * @memberof module:universe + * + * @param dir - base directory to start the search + * @param filter - filtering function to indicate that a file should be selected + */ +async function findFiles( + dir: string, + filter: (filePath: string, stat: Stats) => boolean = () => true, +): Promise { + const allFiles = await fs.readdir(dir, { recursive: true }); + const filePaths: string[] = []; + for (const fileName of allFiles) { + const filePath = path.join(dir, fileName); + const stat = await fs.stat(filePath); + if (!stat.isDirectory() && filter(filePath, stat)) { + filePaths.push(filePath); + } + } + + return filePaths; +} + +interface FromFsOptions { + /** + * Additional filters to apply to json files when loading a universe from FS + */ + filter: (filePath: string, stat: Stats) => boolean; + + /** + * Only load the canonical version of the contract and ignore + * aliases + */ + canonicalOnly: boolean; +} + +export class Universe extends Contract { + constructor() { + super({ type: UNIVERSE }); + } + + /** + * @summary recursively looks up all json files under a directory and adds them + * to the universe + * @function + * @static + * @name module:contrato.Universe.fromFs + * @public + * + * @param dir full path of the directory to load + * @param options additional configuration for the search and build process + * + */ + static async fromFs( + dir: string, + { filter = () => true, canonicalOnly = false }: Partial = {}, + ): Promise { + const allFiles = await findFiles( + dir, + (filePath, stat) => + path.extname(filePath) === '.json' && filter(filePath, stat), + ); + + const universe = new Universe(); + const children: Contract[] = []; + await concurrentForEach( + allFiles.values(), + async (file) => { + const contents = await fs.readFile(file, { encoding: 'utf8' }); + let source = JSON.parse(contents); + + if (canonicalOnly) { + // Ignore aliases + const { aliases, ...obj } = source; + source = obj; + } + + children.push(...Contract.build(source)); + }, + 10, + ); + + universe.addChildren(children); + + return universe; + } +} + +export default Universe; diff --git a/tests/universe/contracts/arch.sw/aarch64/contract.json b/tests/universe/contracts/arch.sw/aarch64/contract.json new file mode 100644 index 0000000..e108e79 --- /dev/null +++ b/tests/universe/contracts/arch.sw/aarch64/contract.json @@ -0,0 +1,15 @@ +{ + "slug": "aarch64", + "version": "1", + "type": "arch.sw", + "name": "ARM v8", + "data": { "arch": "aarch64" }, + "requires": [ + { + "type": "hw.device-type", + "data": { + "arch": "aarch64" + } + } + ] +} diff --git a/tests/universe/contracts/arch.sw/amd64/contract.json b/tests/universe/contracts/arch.sw/amd64/contract.json new file mode 100644 index 0000000..7ad0a10 --- /dev/null +++ b/tests/universe/contracts/arch.sw/amd64/contract.json @@ -0,0 +1,15 @@ +{ + "slug": "amd64", + "version": "1", + "type": "arch.sw", + "name": "Intel 64-bit (x86-64)", + "data": { "arch": "amd64" }, + "requires": [ + { + "type": "hw.device-type", + "data": { + "arch": "amd64" + } + } + ] +} diff --git a/tests/universe/contracts/arch.sw/armv7hf/contract.json b/tests/universe/contracts/arch.sw/armv7hf/contract.json new file mode 100644 index 0000000..9f90791 --- /dev/null +++ b/tests/universe/contracts/arch.sw/armv7hf/contract.json @@ -0,0 +1,15 @@ +{ + "slug": "armv7hf", + "version": "1", + "type": "arch.sw", + "name": "ARM v7", + "data": { "arch": "armv7hf" }, + "requires": [ + { + "type": "hw.device-type", + "data": { + "arch": "armv7hf" + } + } + ] +} diff --git a/tests/universe/contracts/arch.sw/rpi/contract.json b/tests/universe/contracts/arch.sw/rpi/contract.json new file mode 100644 index 0000000..023c5c1 --- /dev/null +++ b/tests/universe/contracts/arch.sw/rpi/contract.json @@ -0,0 +1,15 @@ +{ + "slug": "rpi", + "version": "1", + "type": "arch.sw", + "name": "ARM v6", + "data": { "arch": "armv6hf" }, + "requires": [ + { + "type": "hw.device-type", + "data": { + "arch": "rpi" + } + } + ] +} diff --git a/tests/universe/contracts/hw.device-type/intel-nuc/contract.json b/tests/universe/contracts/hw.device-type/intel-nuc/contract.json new file mode 100644 index 0000000..5d93ead --- /dev/null +++ b/tests/universe/contracts/hw.device-type/intel-nuc/contract.json @@ -0,0 +1,29 @@ +{ + "slug": "intel-nuc", + "version": "1", + "type": "hw.device-type", + "aliases": [ + "nuc" + ], + "name": "Intel NUC", + "data": { + "arch": "amd64", + "hdmi": true, + "led": false, + "connectivity": { + "bluetooth": true, + "wifi": true + }, + "storage": { + "internal": true + }, + "media": { + "defaultBoot": "internal", + "altBoot": [ + "usb_mass_storage", + "network" + ] + }, + "is_private": false + } +} diff --git a/tests/universe/contracts/hw.device-type/jetson-nano/contract.json b/tests/universe/contracts/hw.device-type/jetson-nano/contract.json new file mode 100644 index 0000000..33c5a97 --- /dev/null +++ b/tests/universe/contracts/hw.device-type/jetson-nano/contract.json @@ -0,0 +1,24 @@ +{ + "slug": "jetson-nano", + "version": "1", + "type": "hw.device-type", + "aliases": [], + "name": "Nvidia Jetson Nano SD-CARD", + "data": { + "arch": "aarch64", + "hdmi": true, + "led": false, + "connectivity": { + "bluetooth": false, + "wifi": false + }, + "storage": { + "internal": false + }, + "flashProtocol": "jetsonFlash", + "media": { + "defaultBoot": "sdcard" + }, + "is_private": false + } +} diff --git a/tests/universe/contracts/hw.device-type/raspberry-pi/contract.json b/tests/universe/contracts/hw.device-type/raspberry-pi/contract.json new file mode 100644 index 0000000..a4f793c --- /dev/null +++ b/tests/universe/contracts/hw.device-type/raspberry-pi/contract.json @@ -0,0 +1,29 @@ +{ + "slug": "raspberry-pi", + "version": "1", + "type": "hw.device-type", + "aliases": [ + "raspberrypi" + ], + "name": "Raspberry Pi (v1 / Zero / Zero W)", + "data": { + "arch": "rpi", + "hdmi": true, + "led": true, + "connectivity": { + "bluetooth": true, + "wifi": true + }, + "storage": { + "internal": false + }, + "media": { + "defaultBoot": "sdcard", + "altBoot": [ + "usb_mass_storage", + "network" + ] + }, + "is_private": false + } +} diff --git a/tests/universe/contracts/hw.device-type/raspberry-pi2/contract.json b/tests/universe/contracts/hw.device-type/raspberry-pi2/contract.json new file mode 100644 index 0000000..b6be28d --- /dev/null +++ b/tests/universe/contracts/hw.device-type/raspberry-pi2/contract.json @@ -0,0 +1,29 @@ +{ + "slug": "raspberry-pi2", + "version": "1", + "type": "hw.device-type", + "aliases": [ + "raspberrypi2" + ], + "name": "Raspberry Pi 2", + "data": { + "arch": "armv7hf", + "hdmi": true, + "led": true, + "connectivity": { + "bluetooth": false, + "wifi": false + }, + "storage": { + "internal": false + }, + "media": { + "defaultBoot": "sdcard", + "altBoot": [ + "usb_mass_storage", + "network" + ] + }, + "is_private": false + } +} diff --git a/tests/universe/contracts/hw.device-type/raspberrypi3/contract.json b/tests/universe/contracts/hw.device-type/raspberrypi3/contract.json new file mode 100644 index 0000000..61169fd --- /dev/null +++ b/tests/universe/contracts/hw.device-type/raspberrypi3/contract.json @@ -0,0 +1,27 @@ +{ + "slug": "raspberrypi3", + "version": "1", + "type": "hw.device-type", + "aliases": [], + "name": "Raspberry Pi 3", + "data": { + "arch": "armv7hf", + "hdmi": true, + "led": true, + "connectivity": { + "bluetooth": true, + "wifi": true + }, + "storage": { + "internal": false + }, + "media": { + "defaultBoot": "sdcard", + "altBoot": [ + "usb_mass_storage", + "network" + ] + }, + "is_private": false + } +} diff --git a/tests/universe/contracts/hw.device-type/raspberrypi4-64/contract.json b/tests/universe/contracts/hw.device-type/raspberrypi4-64/contract.json new file mode 100644 index 0000000..05c884f --- /dev/null +++ b/tests/universe/contracts/hw.device-type/raspberrypi4-64/contract.json @@ -0,0 +1,27 @@ +{ + "slug": "raspberrypi4-64", + "version": "1", + "type": "hw.device-type", + "aliases": [], + "name": "Raspberry Pi 4 (using 64bit OS)", + "data": { + "arch": "aarch64", + "hdmi": true, + "led": true, + "connectivity": { + "bluetooth": true, + "wifi": true + }, + "storage": { + "internal": false + }, + "media": { + "defaultBoot": "sdcard", + "altBoot": [ + "usb_mass_storage", + "network" + ] + }, + "is_private": false + } +} diff --git a/tests/universe/from-fs.spec.ts b/tests/universe/from-fs.spec.ts new file mode 100644 index 0000000..a0815d6 --- /dev/null +++ b/tests/universe/from-fs.spec.ts @@ -0,0 +1,130 @@ +import path from 'path'; +import { expect } from '../chai'; + +import Contract from '../../lib/contract'; +import Universe from '../../lib/universe'; + +describe('Universe fromFs', () => { + it('allows loading a universe from a directory', async () => { + const universe = await Universe.fromFs(path.join(__dirname, './contracts')); + + expect( + universe.findChildren( + Contract.createMatcher({ type: 'hw.device-type', slug: 'raspberrypi' }), + ), + ).to.have.lengthOf(1); + expect( + universe.findChildren( + Contract.createMatcher({ + type: 'hw.device-type', + slug: 'raspberry-pi', + }), + ), + ).to.have.lengthOf(1); + expect( + universe.findChildren( + Contract.createMatcher({ + type: 'hw.device-type', + slug: 'intel-nuc', + }), + ), + ).to.have.lengthOf(1); + }); + + it('allows loading only canonical contracts from a directory', async () => { + const universe = await Universe.fromFs( + path.join(__dirname, './contracts'), + { canonicalOnly: true }, + ); + + // The canonical version of the contract should exist + expect( + universe.findChildren( + Contract.createMatcher({ + type: 'hw.device-type', + slug: 'raspberry-pi', + }), + ), + ).to.have.lengthOf(1); + // But the alias should not + expect( + universe.findChildren( + Contract.createMatcher({ + type: 'hw.device-type', + slug: 'raspberrypi', + }), + ), + ).to.have.lengthOf(0); + expect( + universe.findChildren( + Contract.createMatcher({ + type: 'hw.device-type', + slug: 'raspberrypi2', + }), + ), + ).to.have.lengthOf(0); + expect( + universe.findChildren( + Contract.createMatcher({ + type: 'hw.device-type', + slug: 'intel-nuc', + }), + ), + ).to.have.lengthOf(1); + }); + + it('allows filtering files when searching contracts', async () => { + const universe = await Universe.fromFs( + path.join(__dirname, './contracts'), + { + // Only load raspberrypi contracts + filter: (filePath) => + path.basename(path.dirname(filePath)).startsWith('raspberry'), + }, + ); + + // The canonical version of the contract should exist + expect( + universe.findChildren( + Contract.createMatcher({ + type: 'hw.device-type', + slug: 'raspberry-pi', + }), + ), + ).to.have.lengthOf(1); + // But the alias should not + expect( + universe.findChildren( + Contract.createMatcher({ + type: 'hw.device-type', + slug: 'raspberrypi', + }), + ), + ).to.have.lengthOf(1); + expect( + universe.findChildren( + Contract.createMatcher({ + type: 'hw.device-type', + slug: 'raspberrypi2', + }), + ), + ).to.have.lengthOf(1); + // Other contracts should not be loaded to the universe + expect( + universe.findChildren( + Contract.createMatcher({ + type: 'hw.device-type', + slug: 'intel-nuc', + }), + ), + ).to.have.lengthOf(0); + expect( + universe.findChildren( + Contract.createMatcher({ + type: 'hw.device-type', + slug: 'jetson-nano', + }), + ), + ).to.have.lengthOf(0); + }); +}); From 20c9f26832f3cfafd44699e733b8fb4297bf2a35 Mon Sep 17 00:00:00 2001 From: Felipe Lalanne Date: Fri, 10 Jan 2025 12:24:16 -0300 Subject: [PATCH 2/2] Use p-map rather than custom concurrentForEach Change-type: patch --- lib/universe.ts | 68 +++++++++++++------------------------------------ package.json | 1 + tsconfig.json | 9 ++++--- 3 files changed, 25 insertions(+), 53 deletions(-) diff --git a/lib/universe.ts b/lib/universe.ts index 43aeddb..fd852cd 100644 --- a/lib/universe.ts +++ b/lib/universe.ts @@ -5,39 +5,6 @@ import type { Stats } from 'fs'; import Contract from './contract'; import { UNIVERSE } from './types/types'; -/** - * @summary Run the callback function concurrently on elements from the given iterator - * @function - * @memberof module:universe - * - * Gets up to `concurrency` elements from the given iterator and apply the asynchronous function - * concurrently using `Promise.all`. - * - * If at any point the call to the callback fails, the function will throw the error - * - * @param it - iterator of elements to go traverse - * @param callbackFn - a function to execute for each element in the iterator - * @param concurrency - number of elements to apply the function at the same time. Default to 1 - */ -export async function concurrentForEach( - it: IterableIterator, - callbackFn: (t: T) => PromiseLike, - concurrency = 1, -): Promise { - const run = async () => { - const next = it.next(); - if (next.value && !next.done) { - await callbackFn(next.value); - await run(); - } - }; - const runs = []; - for (let i = 0; i < concurrency; i++) { - runs.push(run()); - } - await Promise.all(runs); -} - /** * @summary recursively find all files under the directory that match the given filter * @function @@ -103,24 +70,25 @@ export class Universe extends Contract { path.extname(filePath) === '.json' && filter(filePath, stat), ); - const universe = new Universe(); - const children: Contract[] = []; - await concurrentForEach( - allFiles.values(), - async (file) => { - const contents = await fs.readFile(file, { encoding: 'utf8' }); - let source = JSON.parse(contents); + const { default: pMap } = await import('p-map'); - if (canonicalOnly) { - // Ignore aliases - const { aliases, ...obj } = source; - source = obj; - } - - children.push(...Contract.build(source)); - }, - 10, - ); + const universe = new Universe(); + const children = ( + await pMap( + allFiles, + async (file) => { + const contents = await fs.readFile(file, { encoding: 'utf8' }); + let source = JSON.parse(contents); + if (canonicalOnly) { + // Ignore aliases + const { aliases, ...obj } = source; + source = obj; + } + return Contract.build(source); + }, + { concurrency: 10 }, + ) + ).flat(); universe.addChildren(children); diff --git a/package.json b/package.json index 5c1f9e9..0cc7f88 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "json-schema": "^0.4.0", "lodash": "^4.17.19", "object-hash": "^1.3.1", + "p-map": "^7.0.3", "promised-handlebars": "^2.0.1", "semver": "^5.7.1" }, diff --git a/tsconfig.json b/tsconfig.json index 5c9f405..d44d139 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { - "module": "commonjs", - "esModuleInterop": true, + "module": "Node16", + "moduleResolution": "node16", "outDir": "build", "noUnusedParameters": true, "noUnusedLocals": true, @@ -13,5 +13,8 @@ "skipLibCheck": true, "resolveJsonModule": true }, - "include": ["lib/**/*.ts", "typings/**/*.ts"] + "include": [ + "lib/**/*.ts", + "typings/**/*.ts" + ] }