Merge pull request #589 from terascope/fix-loading-of-lz4

v2.0.2 - Defer loading of lz4 module (to fix error handling)

asset/asset.json

@@ -1,5 +1,5 @@
 {
     "name": "file",
-    "version": "2.0.1",
+    "version": "2.0.2",
     "description": "A set of processors for exporting data to files"
 }

asset/package.json

@@ -1,6 +1,6 @@
 {
     "name": "file",
-    "version": "2.0.1",
+    "version": "2.0.2",
     "description": "A set of processors for working with files",
     "private": true,
     "workspaces": {
@@ -14,7 +14,7 @@
         "build:watch": "yarn build --watch"
     },
     "dependencies": {
-        "@terascope/file-asset-apis": "^0.4.1",
+        "@terascope/file-asset-apis": "^0.4.2",
         "@terascope/job-components": "^0.48.4",
         "csvtojson": "^2.0.10",
         "fs-extra": "^9.1.0",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "file-assets-bundle",
-    "version": "2.0.1",
+    "version": "2.0.2",
     "description": "Teraslice processors for working with data stored in files on disk",
     "repository": "https://github.com/terascope/file-assets.git",
     "author": "Terascope, LLC <[email protected]>",
@@ -17,6 +17,7 @@
         "build:watch": "tsc --build --watch",
         "lint": "eslint --ignore-path .gitignore --ext .js,.ts .",
         "lint:fix": "yarn lint --fix",
+        "setup": "yarn && yarn build --force",
         "test": "jest",
         "test:all": "yarn workspaces run test",
         "test:watch": "jest --coverage=false --notify --watch --onlyChanged",
@@ -26,7 +27,7 @@
     "dependencies": {},
     "devDependencies": {
         "@terascope/eslint-config": "^0.5.5",
-        "@terascope/file-asset-apis": "^0.4.1",
+        "@terascope/file-asset-apis": "^0.4.2",
         "@terascope/job-components": "^0.48.4",
         "@types/fs-extra": "^9.0.9",
         "@types/jest": "^26.0.22",

packages/file-asset-apis/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@terascope/file-asset-apis",
-    "version": "0.4.1",
+    "version": "0.4.2",
     "description": "file reader and sender apis",
     "publishConfig": {
         "access": "public"

packages/file-asset-apis/src/base/Compressor.ts

@@ -1,15 +1,10 @@
 import { gzip, ungzip } from 'node-gzip';
-// @ts-expect-error
-import lz4init from 'lz4-asm/dist/lz4asm';
 import { Compression } from '../interfaces';
-
-const lz4Module = {};
-const lz4Ready = lz4init(lz4Module);
+import { getLZ4 } from './lz4';
 
 async function lz4Compress(data: Buffer|string): Promise<Buffer> {
-    const { lz4js } = await lz4Ready;
-
-    return lz4js.compress(
+    const { compress } = await getLZ4();
+    return compress(
         Buffer.isBuffer(data)
             ? data
             : Buffer.from(data)
@@ -27,8 +22,8 @@ async function noneCompress(data: Buffer|string): Promise<Buffer> {
 }
 
 async function lz4Decompress(data: Buffer|string): Promise<string> {
-    const { lz4js } = await lz4Ready;
-    const buf = lz4js.decompress(data);
+    const { decompress } = await getLZ4();
+    const buf = decompress(data);
     return buf.toString();
 }
 

packages/file-asset-apis/src/base/index.ts

@@ -5,3 +5,4 @@ export * from './Compressor';
 export * from './Formatter';
 export * from './createFileName';
 export * from './slice';
+export * from './lz4';

packages/file-asset-apis/src/base/lz4.ts

@@ -0,0 +1,18 @@
+// @ts-expect-error
+import lz4init from 'lz4-asm/dist/lz4asm';
+
+export interface LZ4 {
+    compress(data: Buffer): Promise<Buffer>;
+    decompress(data: Buffer|string): Buffer;
+}
+
+const lz4Module = {};
+let lz4js: LZ4|undefined;
+
+export async function getLZ4(): Promise<LZ4> {
+    if (lz4js) return lz4js;
+
+    const lz4Ready = lz4init(lz4Module);
+    lz4js = (await lz4Ready).lz4js;
+    return lz4js!;
+}