Wr/retrieve nuts (#78)

* chore: retrieve NUTs

.circleci/config.yml

@@ -16,12 +16,14 @@ workflows:
           name: node-12
       - release-management/test-nut:
           name: nuts-on-linux
+          node_version: lts
           sfdx_version: latest
           requires:
             - node-latest
       - release-management/test-nut:
           name: nuts-on-windows
           sfdx_version: latest
+          node_version: lts
           os: windows
           requires:
             - node-latest

CHANGELOG.md

@@ -6,26 +6,23 @@ All notable changes to this project will be documented in this file. See [standa
 
 ### [0.0.14](https://github.com/salesforcecli/plugin-source/compare/v0.0.13...v0.0.14) (2021-05-04)
 
-
 ### Bug Fixes
 
-* support hooks for deployRecentValidation and async deploys ([#75](https://github.com/salesforcecli/plugin-source/issues/75)) ([ac787f9](https://github.com/salesforcecli/plugin-source/commit/ac787f9f8238bd09d44878a6c9994384a5c567c7))
+- support hooks for deployRecentValidation and async deploys ([#75](https://github.com/salesforcecli/plugin-source/issues/75)) ([ac787f9](https://github.com/salesforcecli/plugin-source/commit/ac787f9f8238bd09d44878a6c9994384a5c567c7))
 
 ### [0.0.13](https://github.com/salesforcecli/plugin-source/compare/v0.0.12...v0.0.13) (2021-05-04)
 
-
 ### Bug Fixes
 
-* deploy output fixes ([#74](https://github.com/salesforcecli/plugin-source/issues/74)) ([d1bb8be](https://github.com/salesforcecli/plugin-source/commit/d1bb8be605458aea81503c1e5bc6974fe03c0ec6))
+- deploy output fixes ([#74](https://github.com/salesforcecli/plugin-source/issues/74)) ([d1bb8be](https://github.com/salesforcecli/plugin-source/commit/d1bb8be605458aea81503c1e5bc6974fe03c0ec6))
 
 ### [0.0.12](https://github.com/salesforcecli/plugin-source/compare/v0.0.11...v0.0.12) (2021-05-01)
 
 ### [0.0.11](https://github.com/salesforcecli/plugin-source/compare/v0.0.10...v0.0.11) (2021-04-29)
 
-
 ### Bug Fixes
 
-* add NUT testing LWC bug ([#70](https://github.com/salesforcecli/plugin-source/issues/70)) ([d6cb456](https://github.com/salesforcecli/plugin-source/commit/d6cb456cdf127e7e896511b8af7606fce25973cb))
+- add NUT testing LWC bug ([#70](https://github.com/salesforcecli/plugin-source/issues/70)) ([d6cb456](https://github.com/salesforcecli/plugin-source/commit/d6cb456cdf127e7e896511b8af7606fce25973cb))
 
 ### [0.0.10](https://github.com/salesforcecli/plugin-source/compare/v0.0.9...v0.0.10) (2021-04-23)
 

package.json

@@ -16,10 +16,9 @@
   "devDependencies": {
     "@oclif/dev-cli": "^1",
     "@oclif/plugin-command-snapshot": "^2.0.0",
-    "@salesforce/cli-plugins-testkit": "^0.0.27",
-    "@salesforce/dev-config": "^2.1.0",
-    "@salesforce/dev-scripts": "^0.9.1",
-    "@salesforce/plugin-apex": "^0.1.18",
+    "@salesforce/cli-plugins-testkit": "^1.1.1",
+    "@salesforce/dev-config": "^2.1.2",
+    "@salesforce/dev-scripts": "^0.9.11",
     "@salesforce/plugin-command-reference": "^1.3.0",
     "@salesforce/plugin-config": "^1.2.6",
     "@salesforce/prettier-config": "^0.0.2",
@@ -34,9 +33,9 @@
     "debug": "^4.3.1",
     "eslint": "^6.8.0",
     "eslint-config-prettier": "^6.11.0",
-    "eslint-config-salesforce": "^0.1.0",
-    "eslint-config-salesforce-license": "^0.1.0",
-    "eslint-config-salesforce-typescript": "^0.2.0",
+    "eslint-config-salesforce": "^0.1.6",
+    "eslint-config-salesforce-license": "^0.1.6",
+    "eslint-config-salesforce-typescript": "^0.2.7",
     "eslint-plugin-header": "^3.0.0",
     "eslint-plugin-import": "^2.20.2",
     "eslint-plugin-jsdoc": "^27.0.3",
@@ -48,6 +47,7 @@
     "nyc": "^15.1.0",
     "prettier": "^2.0.5",
     "pretty-quick": "^2.0.1",
+    "salesforcedx-templates": "^49.8.0",
     "shelljs": "^0.8.4",
     "shx": "0.3.3",
     "sinon": "^9.0.2",
@@ -83,7 +83,7 @@
       "@oclif/plugin-command-snapshot",
       "@oclif/plugin-help",
       "@salesforce/plugin-command-reference",
-      "@salesforce/plugin-apex",
+      "salesforcedx-templates",
       "@salesforce/plugin-config"
     ],
     "topics": {
@@ -120,8 +120,10 @@
     "test": "sf-test",
     "test:command-reference": "./bin/run commandreference:generate --erroronwarnings",
     "test:deprecation-policy": "./bin/run snapshot:compare",
-    "test:nuts": "ts-node ./test/nuts/generateNuts.ts && nyc mocha \"**/convert.*.nut.ts\" --slow 3000 --timeout 600000 --parallel --retries 0",
-    "test:nuts:convert": "ts-node ./test/nuts/generateNuts.ts && nyc mocha \"**/convert.*.nut.ts\" --slow 3000 --timeout 600000 --parallel --retries 0",
+    "test:nuts": "ts-node ./test/nuts/generateNuts.ts && nyc mocha \"**/retrieve.*.nut.ts\" --slow 3000 --timeout 600000 --retries 0",
+    "test:nuts:convert": "PLUGIN_SOURCE_SEED_FILTER=\"convert\" ts-node ./test/nuts/generateNuts.ts && nyc mocha \"**/*.nut.ts\" --slow 3000 --timeout 600000 --parallel --retries 0",
+    "test:nuts:retrieve": "PLUGIN_SOURCE_SEED_FILTER=\"retrieve\" ts-node ./test/nuts/generateNuts.ts && nyc mocha \"**/*.nut.ts\" --slow 3000 --timeout 600000 --parallel --retries 0",
+    "test:nuts:deploy": "PLUGIN_SOURCE_SEED_FILTER=\"deploy\" ts-node ./test/nuts/generateNuts.ts && nyc mocha \"**/*.nut.ts\" --slow 3000 --timeout 600000 --parallel --retries 0",
     "version": "oclif-dev readme"
   },
   "husky": {

src/commands/force/source/deploy.ts

@@ -116,7 +116,7 @@ export class Deploy extends DeployCommand {
   protected async deploy(): Promise<void> {
     this.isAsync = this.getFlag<Duration>('wait').quantity === 0;
     this.isRest = await this.isRestDeploy();
-    this.log(`*** Deploying with ${this.isRest ? 'REST' : 'SOAP'} API ***`);
+    this.ux.log(`*** Deploying with ${this.isRest ? 'REST' : 'SOAP'} API ***`);
 
     if (this.flags.validateddeployrequestid) {
       this.deployResult = await this.deployRecentValidation();

src/deployCommand.ts

@@ -31,7 +31,7 @@ export abstract class DeployCommand extends SourceCommand {
 
     const res = await this.org.getConnection().metadata.checkDeployStatus(deployId, true);
 
-    const deployStatus = (res as unknown) as MetadataApiDeployStatus;
+    const deployStatus = res as unknown as MetadataApiDeployStatus;
     return new DeployResult(deployStatus, new ComponentSet());
   }
 
@@ -98,14 +98,14 @@ export abstract class DeployCommand extends SourceCommand {
         const deployResult = await this.report(deployId);
         return {
           completed: getBoolean(deployResult, 'response.done'),
-          payload: (deployResult as unknown) as AnyJson,
+          payload: deployResult as unknown as AnyJson,
         };
       },
     };
 
     const pollingOptions = { ...defaultOptions, ...options };
 
     const pollingClient = await PollingClient.create(pollingOptions);
-    return (pollingClient.subscribe() as unknown) as Promise<DeployResult>;
+    return pollingClient.subscribe() as unknown as Promise<DeployResult>;
   }
 }

test/commands/source/testConsts.ts

@@ -18,10 +18,8 @@ export const exampleSourceComponent = {
       adapter: 'matchingContentFile',
     },
   },
-  xml:
-    '/Users/william.ruemmele/projects/scratches/dreamhouse-lwc/force-app/main/default/classes/GeocodingService.cls-meta.xml',
-  content:
-    '/Users/william.ruemmele/projects/scratches/dreamhouse-lwc/force-app/main/default/classes/GeocodingService.cls',
+  xml: '/dreamhouse-lwc/force-app/main/default/classes/GeocodingService.cls-meta.xml',
+  content: '/dreamhouse-lwc/force-app/main/default/classes/GeocodingService.cls',
 };
 
 export const exampleDeployResponse = {

test/nuts/assertions.ts

@@ -15,7 +15,7 @@ import { MetadataResolver } from '@salesforce/source-deploy-retrieve';
 import { debug, Debugger } from 'debug';
 import { ApexClass, ApexTestResult, Context, SourceMember, SourceState, StatusResult } from './types';
 import { ExecutionLog } from './executionLog';
-import { FileTracker, countFiles } from './fileTracker';
+import { countFiles, FileTracker } from './fileTracker';
 
 use(chaiEach);
 
@@ -59,11 +59,16 @@ export class Assertions {
   /**
    * Expect all files found by globs to be changed according to the file history provided by FileTracker
    */
-  public async filesToBeChanged(globs: string[]): Promise<void> {
-    const files = await this.doGlob(globs);
-    const fileHistories = files
+  public async filesToBeChanged(globs: string[], ignore: string[] = []): Promise<void> {
+    const all = await this.doGlob(globs);
+    // don't assert a result if nothing is to be ignored
+    const toIgnore = await this.doGlob(ignore, false);
+    const toTrack = all.filter((file) => !toIgnore.includes(file));
+    const fileHistories = toTrack
+      // StaticResource types are inconsistently changed
       .filter((f) => !f.endsWith('.resource-meta.xml'))
-      .map((f) => this.fileTracker.getLatest(f))
+      .filter((f) => !f.endsWith('.resource'))
+      .map((f) => this.fileTracker.getLatest(path.normalize(f)))
       .filter((f) => !!f);
     const allChanged = fileHistories.every((f) => f.changedFromPrevious);
     expect(allChanged, 'all files to be changed').to.be.true;
@@ -72,9 +77,11 @@ export class Assertions {
   /**
    * Expect all files found by globs to NOT be changed according to the file history provided by FileTracker
    */
-  public async filesToNotBeChanged(globs: string[]): Promise<void> {
-    const files = await this.doGlob(globs);
-    const fileHistories = files
+  public async filesToNotBeChanged(globs: string[], ignore: string[] = []): Promise<void> {
+    const all = await this.doGlob(globs);
+    const toIgnore = await this.doGlob(ignore, false);
+    const toTrack = all.filter((file) => !toIgnore.includes(file));
+    const fileHistories = toTrack
       .filter((f) => !f.endsWith('.resource-meta.xml'))
       .map((f) => this.fileTracker.getLatest(f))
       .filter((f) => !!f);
@@ -384,14 +391,22 @@ export class Assertions {
   private async retrieveApexClasses(classNames?: string[]): Promise<ApexClass[]> {
     const query = 'SELECT Name,Id FROM ApexClass';
     const result = await this.connection.tooling.query<ApexClass>(query, { autoFetch: true, maxFetch: 50000 });
-    const apexClasses = classNames ? result.records.filter((r) => classNames.includes(r.Name)) : result.records;
-    return apexClasses;
+    return classNames ? result.records.filter((r) => classNames.includes(r.Name)) : result.records;
   }
 
   private async doGlob(globs: string[], assert = true): Promise<string[]> {
     const files: string[] = [];
-    for (const glob of globs) {
-      const fullGlob = glob.startsWith(this.projectDir) ? glob : [this.projectDir, glob].join('/');
+    const dir = this.projectDir.replace(/\\/g, '/');
+
+    for (let glob of globs) {
+      let fullGlob = glob.replace(/\\/g, '/');
+      if (glob.startsWith('!')) {
+        glob = glob.substr(1);
+        fullGlob = glob.startsWith(dir) ? `!${glob}` : [`!${dir}`, glob].join('/');
+      } else {
+        fullGlob = glob.startsWith(dir) ? glob : [dir, glob].join('/');
+      }
+
       this.debug(`Finding files using glob: ${fullGlob}`);
       const globResults = await fg(fullGlob);
       this.debug('Found: %O', globResults);

test/nuts/generateNuts.ts

@@ -7,19 +7,19 @@
 
 import * as path from 'path';
 import * as os from 'os';
+import * as shelljs from 'shelljs';
 import { fs } from '@salesforce/core';
-import { EXECUTABLES, TEST_REPOS_MAP, RepoConfig } from './testMatrix';
+import { EXECUTABLES, RepoConfig, TEST_REPOS_MAP } from './testMatrix';
 
 const SEED_FILTER = process.env.PLUGIN_SOURCE_SEED_FILTER || '';
 
 async function getSeedFiles(): Promise<string[]> {
   const seedDir = path.join(__dirname, 'seeds');
   const files = await fs.readdir(seedDir);
-  const seeds = files
+  return files
     .filter((f) => f.endsWith('.seed.ts'))
     .filter((f) => f.includes(SEED_FILTER))
     .map((f) => path.resolve(path.join(seedDir, f)));
-  return seeds;
 }
 
 function parseRepoName(repo?: RepoConfig): string {
@@ -54,7 +54,7 @@ async function generateNut(
 
 async function generateNuts(): Promise<void> {
   const generatedDir = path.resolve(__dirname, 'generated');
-  fs.rmSync(generatedDir, { force: true, recursive: true });
+  shelljs.rm('-rf', generatedDir);
   await fs.mkdirp(generatedDir);
   const seeds = await getSeedFiles();
   for (const seed of seeds) {

test/nuts/nutshell.ts

@@ -18,6 +18,7 @@ import { AuthInfo, Connection, fs, NamedPackageDir, SfdxProject } from '@salesfo
 import { AsyncCreatable } from '@salesforce/kit';
 import { debug, Debugger } from 'debug';
 import { MetadataResolver } from '@salesforce/source-deploy-retrieve';
+import * as shelljs from 'shelljs';
 import { Result, StatusResult } from './types';
 import { Assertions } from './assertions';
 import { ExecutionLog } from './executionLog';
@@ -215,7 +216,7 @@ export class Nutshell extends AsyncCreatable<Nutshell.Options> {
    */
   public async readMaxRevision(): Promise<{ sourceMembers: JsonMap }> {
     const maxRevisionPath = path.join(this.session.project.dir, '.sfdx', 'orgs', this.username, 'maxRevision.json');
-    return (fs.readJson(maxRevisionPath) as unknown) as { sourceMembers: JsonMap };
+    return fs.readJson(maxRevisionPath) as unknown as { sourceMembers: JsonMap };
   }
 
   /**
@@ -297,7 +298,7 @@ export class Nutshell extends AsyncCreatable<Nutshell.Options> {
     const allFiles = await this.doGlob(globs);
 
     for (const file of allFiles) {
-      await this.modifyLocalFile(file);
+      await this.modifyLocalFile(path.normalize(file));
     }
   }
 
@@ -370,9 +371,31 @@ export class Nutshell extends AsyncCreatable<Nutshell.Options> {
     await this.writeMaxRevision(maxRevision);
   }
 
+  public isSourcePlugin(): boolean {
+    return this.executable.endsWith(`${path.sep}bin${path.sep}run`);
+  }
+
+  // SDR does not output the package.xml in the same location as toolbelt
+  // so we have to find it within the output dir, move it, and delete the
+  // generated dir.
+  public findAndMoveManifest(dir: string): void {
+    const manifest = shelljs.find(dir).filter((file) => file.endsWith('package.xml'));
+    if (!manifest?.length) {
+      throw Error(`Did not find package.xml within ${dir}`);
+    }
+    shelljs.mv(manifest[0], path.join(process.cwd()));
+    shelljs.rm('-rf', dir);
+  }
+
   protected async init(): Promise<void> {
     if (!Nutshell.Env.getString('TESTKIT_HUB_USERNAME') && !Nutshell.Env.getString('TESTKIT_AUTH_URL')) {
-      ensureString(Nutshell.Env.getString('TESTKIT_JWT_KEY'));
+      const jwtKey = ensureString(Nutshell.Env.getString('TESTKIT_JWT_KEY'));
+      if (!jwtKey) {
+        const err = Error('must set on of : TESTKIT_HUB_USERNAME, TESTKIT_AUTH_RUL, or TESTKIT_JWT_KEY');
+        err.name = 'InvalidTestEnvironment';
+        throw err;
+      }
+      ensureString(jwtKey);
       ensureString(Nutshell.Env.getString('TESTKIT_JWT_CLIENT_ID'));
       ensureString(Nutshell.Env.getString('TESTKIT_HUB_INSTANCE'));
     }
@@ -456,7 +479,7 @@ export class Nutshell extends AsyncCreatable<Nutshell.Options> {
       ? []
       : [
           'sfdx config:set restDeploy=false --global',
-          'sfdx force:org:create -d 1 -s -f config/project-scratch-def.json',
+          `sfdx force:org:create -d 1 -s -f ${path.join('config', 'project-scratch-def.json')}`,
         ];
     return await TestSession.create({
       project: { gitClone: this.repository },
@@ -479,9 +502,16 @@ export class Nutshell extends AsyncCreatable<Nutshell.Options> {
   }
 
   private async doGlob(globs: string[]): Promise<string[]> {
-    const fullGlobs = globs.map((g) =>
-      g.startsWith(this.session.project.dir) ? g : [this.session.project.dir, g].join('/')
-    );
+    const dir = this.session.project.dir.replace(/\\/g, '/');
+    const fullGlobs = globs.map((g) => {
+      g = g.replace(/\\/g, '/');
+      if (g.startsWith('!')) {
+        g = g.substr(1).startsWith(dir) ? `!${g}` : [`!${dir}`, g].join('/');
+      } else {
+        g = g.startsWith(dir) ? g : [dir, g].join('/');
+      }
+      return g;
+    });
     return fg(fullGlobs);
   }
 }

test/nuts/seeds/convert.seed.ts

@@ -15,22 +15,6 @@ import { TEST_REPOS_MAP } from '../testMatrix';
 const REPO = TEST_REPOS_MAP.get('%REPO_URL%');
 const EXECUTABLE = '%EXECUTABLE%';
 
-// SDR does not output the package.xml in the same location as toolbelt
-// so we have to find it within the output dir, move it, and delete the
-// generated dir.
-const mvManifest = (dir: string) => {
-  const manifest = shelljs.find(dir).filter((file) => file.endsWith('package.xml'));
-  if (!manifest?.length) {
-    throw Error(`Did not find package.xml within ${dir}`);
-  }
-  shelljs.mv(manifest[0], path.join(process.cwd()));
-  shelljs.rm('-rf', dir);
-};
-
-const isSourcePlugin = (): boolean => {
-  return EXECUTABLE.endsWith(`${path.sep}bin${path.sep}run`);
-};
-
 context('Convert NUTs [name: %REPO_NAME%] [exec: %EXECUTABLE%]', () => {
   let nutshell: Nutshell;
 
@@ -59,7 +43,7 @@ context('Convert NUTs [name: %REPO_NAME%] [exec: %EXECUTABLE%]', () => {
           exitCode: 0,
         });
         const outputDir = path.join(process.cwd(), 'out1');
-        mvManifest(outputDir);
+        nutshell.findAndMoveManifest(outputDir);
         const packageXml = path.join(process.cwd(), 'package.xml');
 
         const res = await nutshell.convert({ args: `--manifest ${packageXml} --outputdir out2`, exitCode: 0 });
@@ -105,7 +89,7 @@ context('Convert NUTs [name: %REPO_NAME%] [exec: %EXECUTABLE%]', () => {
 
     it('should throw an error if the metadata is not valid', async () => {
       const convert = await nutshell.convert({ args: '--metadata DOES_NOT_EXIST', exitCode: 1 });
-      const expectedError = isSourcePlugin() ? 'RegistryError' : 'UnsupportedType';
+      const expectedError = nutshell.isSourcePlugin() ? 'RegistryError' : 'UnsupportedType';
       nutshell.expect.errorToHaveName(convert, expectedError);
     });
   });
@@ -133,7 +117,7 @@ context('Convert NUTs [name: %REPO_NAME%] [exec: %EXECUTABLE%]', () => {
 
     it('should throw an error if the sourcepath is not valid', async () => {
       const convert = await nutshell.convert({ args: '--sourcepath DOES_NOT_EXIST', exitCode: 1 });
-      const expectedError = isSourcePlugin() ? 'SfdxError' : 'SourcePathInvalid';
+      const expectedError = nutshell.isSourcePlugin() ? 'SfdxError' : 'SourcePathInvalid';
       nutshell.expect.errorToHaveName(convert, expectedError);
     });
   });