expand glob paths

nrwl · Aug 2, 2024 · 5b346fa · 5b346fa
1 parent a8da4dd
commit 5b346fa
Show file tree

Hide file tree

Showing 3 changed files with 105 additions and 5 deletions.
diff --git a/workflow-steps/cache/hashing-utils.ts b/workflow-steps/cache/hashing-utils.ts
@@ -1,5 +1,3 @@
-import * as string_decoder from 'string_decoder';
-
 const fs = require('fs');
 const crypto = require('crypto');
 import { glob } from 'glob';
@@ -49,3 +47,36 @@ export function hashKey(key: string): string {
 function hash(input: string) {
   return crypto.createHash('sha256').update(input).digest('hex');
 }
+
+export function buildCachePaths(inputPaths: string) {
+  const directories = Array.from(
+    new Set(
+      inputPaths
+        .split('\n')
+        .filter((p) => p)
+        .reduce(
+          (allPaths, currPath) => [...allPaths, ...expandPath(currPath)],
+          [],
+        ),
+    ),
+  );
+
+  const invalidDirectories = directories.filter(
+    (dir) => !fs.existsSync(dir) || !fs.statSync(dir).isDirectory(),
+  );
+  if (invalidDirectories.length > 0) {
+    throw `The following paths are not valid directories:\n${invalidDirectories.join(
+      '\n',
+    )}`;
+  }
+  return directories;
+}
+
+function expandPath(pattern: string): string[] {
+  const globExpandedPaths = glob.sync(pattern);
+  if (globExpandedPaths.length == 0) {
+    // it's probably not a valid path so we return it so it can be included in the error above
+    return [pattern];
+  }
+  return globExpandedPaths;
+}
diff --git a/workflow-steps/cache/hasing-utils.spec.ts b/workflow-steps/cache/hasing-utils.spec.ts
@@ -1,4 +1,4 @@
-import { hashKey } from './hashing-utils';
+import { buildCachePaths, hashKey } from './hashing-utils';
 import * as path from 'path';
 
 describe('hashing-utils', () => {
@@ -27,4 +27,73 @@ describe('hashing-utils', () => {
       '226f813c92638665c8daa0920cfb83e5f33732f8843042deee348032a1abee40',
     );
   });
+
+  it('should validate simple dirs', () => {
+    let input = `test-files/packages/app1`;
+    let expected = [`test-files/packages/app1`];
+    expect(buildCachePaths(input)).toEqual(expected);
+
+    input = `test-files/packages/app2\ntest-files/packages/app3\n\n`;
+    expected = [`test-files/packages/app2`, `test-files/packages/app3`];
+    expect(buildCachePaths(input)).toEqual(expected);
+  });
+
+  it('should throw when invalid dirs are specified', () => {
+    let input = `test-files/packages/app1\ntest-files/yarn.lock`;
+    expect(() => buildCachePaths(input)).toThrow(
+      'The following paths are not valid directories:\n' +
+        'test-files/yarn.lock',
+    );
+
+    input = `test-files/yarn.lock\ntest-files/main.js`;
+    expect(() => buildCachePaths(input)).toThrow(
+      'The following paths are not valid directories:\n' +
+        'test-files/yarn.lock\n' +
+        'test-files/main.js',
+    );
+
+    input = `test-files/packages/app6`;
+    expect(() => buildCachePaths(input)).toThrow(
+      'The following paths are not valid directories:\n' +
+        'test-files/packages/app6',
+    );
+
+    input = `test-files/packages/app2\ntest-files/packages/app7\n\n`;
+    expect(() => buildCachePaths(input)).toThrow(
+      'The following paths are not valid directories:\n' +
+        'test-files/packages/app7',
+    );
+  });
+
+  it('should support glob paths', () => {
+    let input = `test-files/packages/*/mock_node_modules`;
+    let expected = [
+      `test-files/packages/app1/mock_node_modules`,
+      `test-files/packages/app2/mock_node_modules`,
+      `test-files/packages/app3/mock_node_modules`,
+    ];
+    expect(buildCachePaths(input)).toEqual(expected);
+
+    // it should filter out duplicates
+    input = `test-files/packages/app1/mock_node_modules\ntest-files/packages/*/mock_node_modules\ntest-files/packages`;
+    expected = [
+      `test-files/packages/app1/mock_node_modules`,
+      `test-files/packages/app2/mock_node_modules`,
+      `test-files/packages/app3/mock_node_modules`,
+      `test-files/packages`,
+    ];
+    expect(buildCachePaths(input)).toEqual(expected);
+  });
+
+  it('should filter out duplicates', () => {
+    const input = `test-files/packages/app1\ntest-files/packages/app1/mock_node_modules\ntest-files/packages/*/mock_node_modules\ntest-files/packages`;
+    const expected = [
+      `test-files/packages/app1`,
+      `test-files/packages/app1/mock_node_modules`,
+      `test-files/packages/app2/mock_node_modules`,
+      `test-files/packages/app3/mock_node_modules`,
+      `test-files/packages`,
+    ];
+    expect(buildCachePaths(input)).toEqual(expected);
+  });
 });
diff --git a/workflow-steps/cache/post.ts b/workflow-steps/cache/post.ts
@@ -2,7 +2,7 @@ import { createPromiseClient } from '@bufbuild/connect';
 import { createConnectTransport } from '@bufbuild/connect-web';
 import { CacheService } from './generated_protos/cache_connect';
 import { StoreRequest, StoreResponse } from './generated_protos/cache_pb';
-import { hashKey } from './hashing-utils';
+import { buildCachePaths, hashKey } from './hashing-utils';
 
 const input_key = process.env.NX_CLOUD_INPUT_key;
 const input_paths = process.env.NX_CLOUD_INPUT_paths;
@@ -25,7 +25,7 @@ if (!!cacheWasHit) {
     throw new Error('No cache restore key or paths provided.');
   }
   const key = hashKey(input_key);
-  const paths = input_paths.split('\n').filter((p) => p);
+  const paths = buildCachePaths(input_paths);
 
   cacheClient
     .storeV2(