diff --git a/asset/asset.json b/asset/asset.json index 6e259416..cc7c929f 100644 --- a/asset/asset.json +++ b/asset/asset.json @@ -1,5 +1,5 @@ { "name": "file", - "version": "2.0.1", + "version": "2.0.2", "description": "A set of processors for exporting data to files" } diff --git a/asset/package.json b/asset/package.json index 5ea51bf3..f2060ebc 100644 --- a/asset/package.json +++ b/asset/package.json @@ -1,6 +1,6 @@ { "name": "file", - "version": "2.0.1", + "version": "2.0.2", "description": "A set of processors for working with files", "private": true, "workspaces": { @@ -14,7 +14,7 @@ "build:watch": "yarn build --watch" }, "dependencies": { - "@terascope/file-asset-apis": "^0.4.1", + "@terascope/file-asset-apis": "^0.4.2", "@terascope/job-components": "^0.48.4", "csvtojson": "^2.0.10", "fs-extra": "^9.1.0", diff --git a/package.json b/package.json index daf031be..5998f9ab 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "file-assets-bundle", - "version": "2.0.1", + "version": "2.0.2", "description": "Teraslice processors for working with data stored in files on disk", "repository": "https://github.com/terascope/file-assets.git", "author": "Terascope, LLC ", @@ -17,6 +17,7 @@ "build:watch": "tsc --build --watch", "lint": "eslint --ignore-path .gitignore --ext .js,.ts .", "lint:fix": "yarn lint --fix", + "setup": "yarn && yarn build --force", "test": "jest", "test:all": "yarn workspaces run test", "test:watch": "jest --coverage=false --notify --watch --onlyChanged", @@ -26,7 +27,7 @@ "dependencies": {}, "devDependencies": { "@terascope/eslint-config": "^0.5.5", - "@terascope/file-asset-apis": "^0.4.1", + "@terascope/file-asset-apis": "^0.4.2", "@terascope/job-components": "^0.48.4", "@types/fs-extra": "^9.0.9", "@types/jest": "^26.0.22", diff --git a/packages/file-asset-apis/package.json b/packages/file-asset-apis/package.json index 69c1a38c..1f29a723 100644 --- a/packages/file-asset-apis/package.json +++ b/packages/file-asset-apis/package.json @@ -1,6 +1,6 @@ { "name": "@terascope/file-asset-apis", - "version": "0.4.1", + "version": "0.4.2", "description": "file reader and sender apis", "publishConfig": { "access": "public" diff --git a/packages/file-asset-apis/src/base/Compressor.ts b/packages/file-asset-apis/src/base/Compressor.ts index 3e357fc8..a3affb0d 100644 --- a/packages/file-asset-apis/src/base/Compressor.ts +++ b/packages/file-asset-apis/src/base/Compressor.ts @@ -1,15 +1,10 @@ import { gzip, ungzip } from 'node-gzip'; -// @ts-expect-error -import lz4init from 'lz4-asm/dist/lz4asm'; import { Compression } from '../interfaces'; - -const lz4Module = {}; -const lz4Ready = lz4init(lz4Module); +import { getLZ4 } from './lz4'; async function lz4Compress(data: Buffer|string): Promise { - const { lz4js } = await lz4Ready; - - return lz4js.compress( + const { compress } = await getLZ4(); + return compress( Buffer.isBuffer(data) ? data : Buffer.from(data) @@ -27,8 +22,8 @@ async function noneCompress(data: Buffer|string): Promise { } async function lz4Decompress(data: Buffer|string): Promise { - const { lz4js } = await lz4Ready; - const buf = lz4js.decompress(data); + const { decompress } = await getLZ4(); + const buf = decompress(data); return buf.toString(); } diff --git a/packages/file-asset-apis/src/base/index.ts b/packages/file-asset-apis/src/base/index.ts index 59328173..763a35c1 100644 --- a/packages/file-asset-apis/src/base/index.ts +++ b/packages/file-asset-apis/src/base/index.ts @@ -5,3 +5,4 @@ export * from './Compressor'; export * from './Formatter'; export * from './createFileName'; export * from './slice'; +export * from './lz4'; diff --git a/packages/file-asset-apis/src/base/lz4.ts b/packages/file-asset-apis/src/base/lz4.ts new file mode 100644 index 00000000..f57fd183 --- /dev/null +++ b/packages/file-asset-apis/src/base/lz4.ts @@ -0,0 +1,18 @@ +// @ts-expect-error +import lz4init from 'lz4-asm/dist/lz4asm'; + +export interface LZ4 { + compress(data: Buffer): Promise; + decompress(data: Buffer|string): Buffer; +} + +const lz4Module = {}; +let lz4js: LZ4|undefined; + +export async function getLZ4(): Promise { + if (lz4js) return lz4js; + + const lz4Ready = lz4init(lz4Module); + lz4js = (await lz4Ready).lz4js; + return lz4js!; +}