Merge pull request #230 from forcedotcom/sh/sfdxProject-packageDirs

WIP: Sh/sfdx project package dirs

DEVELOPING.md

@@ -1,6 +1,6 @@
 ## Pre-requisites
 
-1.  We are using Node 8. If you need to work with multiple versions of Node, you
+1.  We are using Node 10 LTS. If you need to work with multiple versions of Node, you
     might consider using [nvm](https://github.com/creationix/nvm).
 1.  This repository uses [yarn](https://yarnpkg.com/) to manage node dependencies. Please install yarn globally using `npm install --global yarn`.
 
@@ -17,7 +17,7 @@ You would only do this once after you cloned the repository.
 
 When you are ready to commit
 
-1.  We enforces commit message format. We recommend using [commitizen](https://github.com/commitizen/cz-cli) by installing it with `yarn add --global commitizen` then commit using `git cz` which will prompt you questions to format the commit message.
+1.  We enforce commit message format. We recommend using [commitizen](https://github.com/commitizen/cz-cli) by installing it with `yarn global add commitizen` then commit using `git cz` which will prompt you questions to format the commit message.
 1.  Before commit and push, husky will run several hooks to ensure the message and that everything lints and compiles properly.
 
 ## List of Useful commands
@@ -28,18 +28,18 @@ This compiles the typescript to javascript.
 
 ### `yarn clean`
 
-This cleans all generated files and directories. Run `yarn cleal-all` will also clean up the node_module directories.
+This cleans all generated files and directories. Run `yarn clean-all` to also clean up the node_module directories.
 
 ### `yarn test`
 
 This tests the typescript using ts-node.
 
 ### `yarn lint`
 
-This lists all the typescript. If there are no errors/warnings
-from tslint, then you get a clean output. But, if they are errors from tslint,
+This lints all the typescript. If there are no errors/warnings
+from tslint, then you get clean output. But, if there are errors from tslint,
 you will see a long error that can be confusing – just focus on the tslint
-errors. The results of this is deeper than what the tslint extension in VS Code
+errors. The results of this are deeper than what the tslint extension in VS Code
 does because of [semantic lint
 rules](https://palantir.github.io/tslint/usage/type-checking/) which requires a
 tsconfig.json to be passed to tslint.

messages/config.json

@@ -1,8 +1,10 @@
 {
-    "UnknownConfigKey": "Unknown config name \"%s\"",
-    "InvalidConfigValue": "Invalid config value. %s",
-    "InvalidInstanceUrl": "Specify a valid Salesforce instance URL",
-    "InvalidApiVersion": "Specify a valid Salesforce API version, for example, 42.0",
-    "InvalidBooleanConfigValue": "The config value can only be set to true or false.",
-    "InvalidProjectWorkspace": "This directory does not contain a valid Salesforce DX project"
-}
+  "UnknownConfigKey": "Unknown config name \"%s\"",
+  "InvalidConfigValue": "Invalid config value. %s",
+  "InvalidInstanceUrl": "Specify a valid Salesforce instance URL",
+  "InvalidApiVersion": "Specify a valid Salesforce API version, for example, 42.0",
+  "InvalidBooleanConfigValue": "The config value can only be set to true or false.",
+  "InvalidProjectWorkspace": "This directory does not contain a valid Salesforce DX project",
+  "SchemaValidationWarning": "The config file: %s is not schema valid\nDue to: %s",
+  "SchemaValidationErrorAction": "Check the file: %s for invalid entries"
+}

package.json

@@ -40,6 +40,7 @@
   "dependencies": {
     "@salesforce/bunyan": "^2.0.0",
     "@salesforce/kit": "^1.0.0",
+    "@salesforce/schemas": "^1.0.1",
     "@salesforce/ts-types": "^1.0.0",
     "@types/jsforce": "1.9.2",
     "debug": "^3.1.0",

src/config/configFile.ts

@@ -10,11 +10,15 @@ import { constants as fsConstants, Stats as fsStats } from 'fs';
 import { homedir as osHomedir } from 'os';
 import { dirname as pathDirname, join as pathJoin } from 'path';
 import { Global } from '../global';
+import { Logger } from '../logger';
+import { Messages } from '../messages';
 import { SfdxError } from '../sfdxError';
 import { fs } from '../util/fs';
 import { resolveProjectPath } from '../util/internal';
 import { BaseConfigStore, ConfigContents } from './configStore';
 
+Messages.importMessagesDirectory(pathJoin(__dirname));
+
 /**
  * Represents a json config file used to manage settings and state. Global config
  * files are stored in the home directory hidden state folder (.sfdx) and local config
@@ -68,6 +72,13 @@ export class ConfigFile<T extends ConfigFile.Options> extends BaseConfigStore<T>
     return isGlobal ? osHomedir() : await resolveProjectPath();
   }
 
+  // whether file contents have been read
+  protected hasRead = false;
+
+  // Initialized in init
+  protected logger!: Logger;
+  protected messages!: Messages;
+
   // Initialized in create
   private path!: string;
 
@@ -98,14 +109,22 @@ export class ConfigFile<T extends ConfigFile.Options> extends BaseConfigStore<T>
   }
 
   /**
-   * Read the config file and set the config contents. Returns the config contents of the config file.
+   * Read the config file and set the config contents. Returns the config contents of the config file. As an
+   * optimization, files are only read once per process and updated in memory and via `write()`. To force
+   * a read from the filesystem pass `force=true`.
    * **Throws** *{@link SfdxError}{ name: 'UnexpectedJsonFileFormat' }* There was a problem reading or parsing the file.
    * @param [throwOnNotFound = false] Optionally indicate if a throw should occur on file read.
+   * @param [force = false] Optionally force the file to be read from disk even when already read within the process.
    */
-  public async read(throwOnNotFound = false): Promise<ConfigContents> {
+  public async read(throwOnNotFound = false, force = false): Promise<ConfigContents> {
     try {
-      const obj = await fs.readJsonMap(this.getPath());
-      this.setContentsFromObject(obj);
+      // Only need to read config files once.  They are kept up to date
+      // internally and updated persistently via write().
+      if (!this.hasRead || force) {
+        this.logger.info(`Reading config file: ${this.getPath()}`);
+        const obj = await fs.readJsonMap(this.getPath());
+        this.setContentsFromObject(obj);
+      }
       return this.getContents();
     } catch (err) {
       if (err.code === 'ENOENT') {
@@ -115,6 +134,10 @@ export class ConfigFile<T extends ConfigFile.Options> extends BaseConfigStore<T>
         }
       }
       throw err;
+    } finally {
+      // Necessarily set this even when an error happens to avoid infinite re-reading.
+      // To attempt another read, pass `force=true`.
+      this.hasRead = true;
     }
   }
 
@@ -131,6 +154,7 @@ export class ConfigFile<T extends ConfigFile.Options> extends BaseConfigStore<T>
 
     await fs.mkdirp(pathDirname(this.getPath()));
 
+    this.logger.info(`Writing to config file: ${this.getPath()}`);
     await fs.writeJson(this.getPath(), this.toObject());
 
     return this.getContents();
@@ -186,6 +210,7 @@ export class ConfigFile<T extends ConfigFile.Options> extends BaseConfigStore<T>
    * options.throwOnNotFound is true.
    */
   protected async init(): Promise<void> {
+    this.logger = await Logger.child(this.constructor.name);
     const statics = this.constructor as typeof ConfigFile;
     let defaultOptions = {};
     try {
@@ -214,6 +239,8 @@ export class ConfigFile<T extends ConfigFile.Options> extends BaseConfigStore<T>
       configRootFolder = pathJoin(configRootFolder, Global.STATE_FOLDER);
     }
 
+    this.messages = Messages.loadMessages('@salesforce/core', 'config');
+
     this.path = pathJoin(configRootFolder, this.options.filePath ? this.options.filePath : '', this.options.filename);
     await this.read(this.options.throwOnNotFound);
   }

src/config/configStore.ts

@@ -69,7 +69,7 @@ export abstract class BaseConfigStore<T extends BaseConfigStore.Options> extends
   public constructor(options: T) {
     super(options);
     this.options = options;
-    this.setContents(this.options.contents || {});
+    this.setContents(this.options.contents);
   }
 
   /**
@@ -216,7 +216,7 @@ export abstract class BaseConfigStore<T extends BaseConfigStore.Options> extends
   }
 
   // Allows extended classes the ability to override the set method. i.e. maybe they don't want
-  // nexted object set from kit.
+  // nested object set from kit.
   protected setMethod(contents: ConfigContents, key: string, value?: ConfigValue) {
     set(contents, key, value);
   }

src/logger.ts

@@ -23,7 +23,7 @@ import {
   Optional
 } from '@salesforce/ts-types';
 import * as Debug from 'debug';
-import * as EventEmitter from 'events';
+import { EventEmitter } from 'events';
 import * as os from 'os';
 import * as path from 'path';
 import { Writable } from 'stream';

src/schema/validator.ts

@@ -38,7 +38,7 @@ export class SchemaValidator {
    * Creates a new `SchemaValidator` instance given a logger and path to a schema file.
    *
    * @param logger An {@link Logger} instance on which to base this class's logger.
-   * @param schemaPath The path from which the schema with which to validate should be loaded.
+   * @param schemaPath The path to the schema file to load and use for validation.
    */
   public constructor(logger: Logger, private schemaPath: string) {
     this.logger = logger.child('SchemaValidator');

src/sfdxProject.ts

@@ -4,18 +4,55 @@
  * SPDX-License-Identifier: BSD-3-Clause
  * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
-
-import { defaults } from '@salesforce/kit';
+import { sep as pathSep } from 'path';
 
 import { ConfigAggregator } from './config/configAggregator';
 import { ConfigFile } from './config/configFile';
 import { ConfigContents } from './config/configStore';
 
+import { defaults, env } from '@salesforce/kit';
 import { JsonMap } from '@salesforce/ts-types';
-import { SfdxError } from './sfdxError';
+import { SchemaValidator } from './schema/validator';
 import { resolveProjectPath, SFDX_PROJECT_JSON } from './util/internal';
+
+import { SfdxError } from './sfdxError';
 import { sfdc } from './util/sfdc';
 
+export type PackageDirDependency = {
+  package: string;
+  versionNumber?: string;
+  [k: string]: unknown;
+};
+
+export type PackageDir = {
+  ancestorId?: string;
+  ancestorVersion?: string;
+  default?: boolean;
+  definitionFile?: string;
+  dependencies?: PackageDirDependency[];
+  includeProfileUserLicenses?: boolean;
+  package?: string;
+  path: string;
+  postInstallScript?: string;
+  postInstallUrl?: string;
+  releaseNotesUrl?: string;
+  uninstallScript?: string;
+  versionDescription?: string;
+  versionName?: string;
+  versionNumber?: string;
+};
+
+export type ProjectJson = ConfigContents & {
+  packageDirectories: PackageDir[];
+  namespace?: string;
+  sourceApiVersion?: string;
+  sfdcLoginUrl?: string;
+  signupTargetLoginUrl?: string;
+  oauthLocalPort?: number;
+  plugins?: { [k: string]: unknown };
+  packageAliases?: { [k: string]: string };
+};
+
 /**
  * The sfdx-project.json config object. This file determines if a folder is a valid sfdx project.
  *
@@ -57,6 +94,9 @@ export class SfdxProjectJson extends ConfigFile<ConfigFile.Options> {
     if (upperCaseKey) {
       throw SfdxError.create('@salesforce/core', 'core', 'InvalidJsonCasing', [upperCaseKey, this.getPath()]);
     }
+
+    await this.schemaValidate();
+
     return contents;
   }
 
@@ -67,9 +107,15 @@ export class SfdxProjectJson extends ConfigFile<ConfigFile.Options> {
       throw SfdxError.create('@salesforce/core', 'core', 'InvalidJsonCasing', [upperCaseKey, this.getPath()]);
     }
 
+    await this.schemaValidate();
+
     return super.write(newContents);
   }
 
+  public getContents(): ProjectJson {
+    return super.getContents() as ProjectJson;
+  }
+
   public getDefaultOptions(options?: ConfigFile.Options): ConfigFile.Options {
     const defaultOptions: ConfigFile.Options = {
       isState: false
@@ -78,6 +124,57 @@ export class SfdxProjectJson extends ConfigFile<ConfigFile.Options> {
     Object.assign(defaultOptions, options || {});
     return defaultOptions;
   }
+
+  /**
+   * Validates sfdx-project.json against the schema.
+   *
+   * Set the `SFDX_PROJECT_JSON_VALIDATION` environment variable to `true` to throw an error when schema validation fails.
+   * A warning is logged by default when the file is invalid.
+   *
+   * ***See*** [sfdx-project.schema.json] (https://raw.githubusercontent.com/forcedotcom/schemas/master/schemas/sfdx-project.schema.json)
+   */
+  public async schemaValidate(): Promise<void> {
+    if (!this.hasRead) {
+      // read calls back into this method after necessarily setting this.hasRead=true
+      await this.read();
+    } else {
+      try {
+        const projectJsonSchemaPath = require.resolve('@salesforce/schemas/sfdx-project.schema.json');
+        const validator = new SchemaValidator(this.logger, projectJsonSchemaPath);
+        await validator.load();
+        await validator.validate(this.getContents());
+      } catch (err) {
+        if (env.getBoolean('SFDX_PROJECT_JSON_VALIDATION', false)) {
+          err.name = 'SfdxSchemaValidationError';
+          const sfdxError = SfdxError.wrap(err);
+          sfdxError.actions = [this.messages.getMessage('SchemaValidationErrorAction', [this.getPath()])];
+          throw sfdxError;
+        } else {
+          this.logger.warn(this.messages.getMessage('SchemaValidationWarning', [this.getPath(), err.message]));
+        }
+      }
+    }
+  }
+
+  /**
+   * Returns the `packageDirectories` within sfdx-project.json, first reading
+   * and validating the file if necessary.
+   */
+  public async getPackageDirectories(): Promise<PackageDir[]> {
+    // Ensure sfdx-project.json has first been read and validated.
+    if (!this.hasRead) {
+      await this.read();
+    }
+
+    const contents = this.getContents();
+    const packageDirs: PackageDir[] = contents.packageDirectories.map(packageDir => {
+      // Change packageDir paths to have path separators that match the OS
+      const regex = pathSep === '/' ? /\\/g : /\//g;
+      packageDir.path = packageDir.path.replace(regex, pathSep);
+      return packageDir;
+    });
+    return packageDirs;
+  }
 }
 
 /**
@@ -98,7 +195,13 @@ export class SfdxProject {
    * **Throws** *{@link SfdxError}{ name: 'InvalidProjectWorkspace' }* If the current folder is not located in a workspace.
    */
   public static async resolve(path?: string): Promise<SfdxProject> {
-    return new SfdxProject(await this.resolveProjectPath(path));
+    const _path = path || process.cwd();
+    if (!SfdxProject.instances.has(_path)) {
+      const project = new SfdxProject(await this.resolveProjectPath(_path));
+      SfdxProject.instances.set(_path, project);
+    }
+    // @ts-ignore Because of the pattern above this is guaranteed to return an instance
+    return SfdxProject.instances.get(_path);
   }
 
   /**
@@ -116,6 +219,9 @@ export class SfdxProject {
     return resolveProjectPath(dir);
   }
 
+  // Cache of SfdxProject instances per path.
+  private static instances = new Map<string, SfdxProject>();
+
   private projectConfig: any; // tslint:disable-line:no-any
 
   // Dynamically referenced in retrieveSfdxProjectJson

src/testSetup.ts

@@ -287,6 +287,9 @@ export const stubContext = (testContext: TestContext) => {
   ) {
     const stub: ConfigStub = testContext.configStubs[this.constructor.name] || {};
 
+    // @ts-ignore set this to true to avoid an infinite loop in tests when reading config files.
+    this.hasRead = true;
+
     if (stub.readFn) {
       return await stub.readFn.call(this);
     }