NEW (SFGE) @W-17915919@ (1 of 2) Implemented SFGE config except java_command (#243)

Author: jfeingold35

packages/code-analyzer-sfge-engine/src/config.ts

@@ -4,21 +4,125 @@ import {
 } from "@salesforce/code-analyzer-engine-api";
 import {getMessage} from "./messages";
 
-// TODO: As soon as we start adding config properties, we can stop ignoring this rule.
-// eslint-disable-next-line @typescript-eslint/no-empty-object-type
 export type SfgeEngineConfig = {
+    disable_limit_reached_violations: boolean;
+    java_max_heap_size?: string;
+    java_thread_count: number;
+    java_thread_timeout: number;
 }
 
 export const DEFAULT_SFGE_ENGINE_CONFIG: SfgeEngineConfig = {
+    disable_limit_reached_violations: false,
+    java_max_heap_size: undefined,
+    java_thread_count: 4,
+    java_thread_timeout: 900000
 };
 
 export const SFGE_ENGINE_CONFIG_DESCRIPTION: ConfigDescription = {
     overview: getMessage('ConfigOverview'),
     fieldDescriptions: {
+        // Whether to prevent 'sfge' from throwing LimitReached violations for complex paths.
+        // By default, Salesforce Graph Engine attempts to detect complex paths that might cause OutOfMemory errors,
+        // and throws LimitReached violations for these paths to continue evaluating other paths safely. The allowed
+        // complexity is dynamically calculated based on the max Java heap size available, but in some cases you may
+        // desire to disable this check in addition to increasing java_max_heap_size.
+        disable_limit_reached_violations: {
+            descriptionText: getMessage('ConfigFieldDescription_disable_limit_reached_violations'),
+            valueType: "boolean",
+            defaultValue: DEFAULT_SFGE_ENGINE_CONFIG.disable_limit_reached_violations
+        },
+        // Specifies the maximum size (in bytes) of the Java heap. The specified value is appended to the '-Xmx' Java
+        // command option. The value must be a multiple of 1024, and greater than 2MB. Append the letter 'k' or 'K' to
+        // indicate kilobytes, m or M to indicate megabytes, and g or G to indicate gigabytes. If unspecified, or specified`
+        // as null, then the JVM will dynamically choose a default value at runtime based on system configuration.
+        java_max_heap_size: {
+            descriptionText: getMessage('ConfigFieldDescription_java_max_heap_size'),
+            valueType: "string",
+            defaultValue: null
+        },
+        // Specifies the number of Java threads available for parallel execution. Increasing the thread count allows for
+        // Salesforce Graph Engine to evaluate more paths at the same time.
+        java_thread_count: {
+            descriptionText: getMessage('ConfigFieldDescription_java_thread_count'),
+            valueType: "number",
+            defaultValue: DEFAULT_SFGE_ENGINE_CONFIG.java_thread_count
+        },
+        // Specifies the maximum time (in milliseconds) a specific Java thread may execute before Salesforce Graph Engine
+        // issues a Timeout violation.
+        java_thread_timeout: {
+            descriptionText: getMessage('ConfigFieldDescription_java_thread_timeout'),
+            valueType: "number",
+            defaultValue: DEFAULT_SFGE_ENGINE_CONFIG.java_thread_timeout
+        }
     }
 }
 
+const JAVA_HEAP_SIZE_REGEX: RegExp = /^\d+[kmg]?$/i;
+
 export async function validateAndNormalizeConfig(cve: ConfigValueExtractor): Promise<SfgeEngineConfig> {
-    cve.validateContainsOnlySpecifiedKeys([]);
-    return {};
-}
+    cve.validateContainsOnlySpecifiedKeys(['disable_limit_reached_violations', 'java_max_heap_size', 'java_thread_count', 'java_thread_timeout']);
+    const sfgeConfigValueExtractor: SfgeConfigValueExtractor = new SfgeConfigValueExtractor(cve);
+    return {
+        disable_limit_reached_violations: sfgeConfigValueExtractor.extractBooleanValue('disable_limit_reached_violations'),
+        java_max_heap_size: sfgeConfigValueExtractor.extractJavaMaxHeapSize(),
+        java_thread_count: sfgeConfigValueExtractor.extractNumericValue('java_thread_count'),
+        java_thread_timeout: sfgeConfigValueExtractor.extractNumericValue('java_thread_timeout')
+    };
+}
+
+class SfgeConfigValueExtractor {
+    private readonly delegateExtractor: ConfigValueExtractor;
+
+    public constructor(delegateExtractor: ConfigValueExtractor) {
+        this.delegateExtractor = delegateExtractor;
+    }
+
+    public extractJavaMaxHeapSize(): string | undefined {
+        const javaMaxHeapSize: string | undefined = this.delegateExtractor.extractString('java_max_heap_size', undefined, JAVA_HEAP_SIZE_REGEX);
+
+        if (!javaMaxHeapSize) {
+            return undefined;
+        }
+
+        if (javaMaxHeapSize.toLowerCase().endsWith('g')) {
+            // A value expressed in gigabytes is always fine.
+            return javaMaxHeapSize;
+        }
+        const numericPortion: number = parseInt(javaMaxHeapSize);
+        if (numericPortion < expressTwoMegabytesInRelevantUnit(javaMaxHeapSize)) {
+            throw new Error(getMessage(
+                'InvalidConfigValue',
+                this.delegateExtractor.getFieldPath('java_max_heap_size'),
+                getMessage('InsufficientMemorySpecified')
+            ));
+        }
+
+        const isStrictlyNumeric: boolean = /^\d+$/.test(javaMaxHeapSize);
+        if (isStrictlyNumeric && numericPortion % 1024 !== 0) {
+            throw new Error(getMessage(
+                'InvalidConfigValue',
+                this.delegateExtractor.getFieldPath('java_max_heap_size'),
+                getMessage('InvalidMemoryMultiple')
+            ));
+        }
+        return javaMaxHeapSize;
+    }
+
+    public extractBooleanValue(fieldName: string): boolean {
+        return this.delegateExtractor.extractBoolean(fieldName, DEFAULT_SFGE_ENGINE_CONFIG[fieldName as keyof SfgeEngineConfig] as boolean)!;
+    }
+
+    public extractNumericValue(fieldName: string): number {
+        return this.delegateExtractor.extractNumber(fieldName, DEFAULT_SFGE_ENGINE_CONFIG[fieldName as keyof SfgeEngineConfig] as number)!;
+    }
+}
+
+function expressTwoMegabytesInRelevantUnit(val: string): number {
+    if (val.toLowerCase().endsWith('m')) {
+        return 2;
+    } else if (val.toLowerCase().endsWith('k')) {
+        return 2048; // 2MB === 2048KB
+    } else {
+        return 2 ** 21; // 2MB === 2^21 bytes
+    }
+}

packages/code-analyzer-sfge-engine/src/engine.ts

@@ -15,7 +15,7 @@ import {
 import {JavaCommandExecutor} from '@salesforce/code-analyzer-engine-api/utils';
 import {Clock, RealClock} from './utils';
 import {getMessage} from './messages';
-import {RuntimeSfgeWrapper, SfgeRuleInfo, SfgeRunResult} from "./sfge-wrapper";
+import {RuntimeSfgeWrapper, SfgeRuleInfo, SfgeRunOptions, SfgeRunResult} from "./sfge-wrapper";
 import {SfgeEngineConfig} from "./config";
 
 const SFGE_RELEVANT_FILE_EXTENSIONS = ['.cls'];
@@ -87,11 +87,19 @@ export class SfgeEngine extends Engine {
 
         const relevantFiles: string[] = await this.getRelevantFilesInWorkspace(runOptions.workspace);
 
+        const sfgeRunOptions: SfgeRunOptions = {
+            heapSizeArg: this.config.java_max_heap_size,
+            logFolder: runOptions.logFolder,
+            disableLimitReachedViolations: this.config.disable_limit_reached_violations,
+            threadCount: this.config.java_thread_count,
+            threadTimeout: this.config.java_thread_timeout
+        };
+
         const sfgeResults: SfgeRunResult[] = await this.sfgeWrapper.invokeRunCommand(
             selectedRuleInfoList,
             relevantFiles, // TODO: WHEN WE ADD PATH-START TARGETING, THIS NEEDS TO CHANGE.
             relevantFiles,
-            runOptions.logFolder,
+            sfgeRunOptions,
             (innerPerc: number, message?: string) => this.emitRunRulesProgressEvent(5 + 93*innerPerc/100, message) // 5%-98%
         );
 

packages/code-analyzer-sfge-engine/src/messages.ts

@@ -7,12 +7,42 @@ const MESSAGE_CATALOG : { [key: string]: string } = {
         `To learn more about this configuration, visit:\n` +
         `  [PLACEHOLDER LINK]`,
 
+    ConfigFieldDescription_disable_limit_reached_violations:
+        `Whether to prevent 'sfge' from throwing LimitReached violations for complex paths.\n` +
+        `By default, Salesforce Graph Engine attempts to detect complex paths that might cause OutOfMemory errors,\n` +
+        `and throws LimitReached violations for these paths to continue evaluating other paths safely. The allowed\n` +
+        `complexity is dynamically calculated based on the max Java heap size available, but in some cases you may\n` +
+        `desire to disable this check in addition to increasing java_max_heap_size.`,
+
+    ConfigFieldDescription_java_max_heap_size:
+        `Specifies the maximum size (in bytes) of the Java heap. The specified value is appended to the '-Xmx' Java\n` +
+        `command option. The value must be a multiple of 1024, and greater than 2MB. Append the letter 'k' or 'K' to\n` +
+        `indicate kilobytes, m or M to indicate megabytes, and g or G to indicate gigabytes. If unspecified, or specified\n` +
+        `as null, then the JVM will dynamically choose a default value at runtime based on system configuration.`,
+
+    ConfigFieldDescription_java_thread_count:
+        `Specifies the number of Java threads available for parallel execution. Increasing the thread count allows for\n` +
+        `Salesforce Graph Engine to evaluate more paths at the same time.`,
+
+    ConfigFieldDescription_java_thread_timeout:
+        `Specifies the maximum time (in milliseconds) a specific Java thread may execute before Salesforce Graph Engine\n` +
+        `issues a Timeout violation.`,
+
     DeveloperPreviewRuleNotification:
         `This rule is in "Developer Preview" and is subject to change. %s`,
 
     UnsupportedEngineName:
         `The SfgeEnginePlugin does not support an engine with the name '%s'.`,
 
+    InvalidConfigValue:
+        `The '%s' configuration value is invalid. %s`,
+
+    InsufficientMemorySpecified:
+        `The amount of memory specified must be >=2MB`,
+
+    InvalidMemoryMultiple:
+        `The amount of memory specified in bytes must be divisible by 1024`,
+
     WorkspaceAppearsIncomplete:
         `Specified workspace is missing %d possibly-relevant file(s) from the folder %s. Salesforce Graph Engine may be unable to create a complete graph of your project without all apex files included in your workspace. This may result in incomplete or incorrect results.`,
 

packages/code-analyzer-sfge-engine/src/sfge-wrapper.ts

@@ -62,6 +62,14 @@ const SFCA_REALTIME_START: string = 'SFCA-REALTIME-START';
 const SFCA_REALTIME_END: string = 'SFCA-REALTIME-END';
 const SFGE_ERROR_START: string = 'SfgeErrorStart';
 
+export type SfgeRunOptions = {
+    logFolder: string;
+    disableLimitReachedViolations: boolean;
+    threadCount: number;
+    threadTimeout: number;
+    heapSizeArg?: string;
+};
+
 export class RuntimeSfgeWrapper {
     private readonly javaCommandExecutor: JavaCommandExecutor;
     private readonly logFileName: string;
@@ -100,20 +108,31 @@ export class RuntimeSfgeWrapper {
         }
     }
 
-    public async invokeRunCommand(selectedRuleInfos: SfgeRuleInfo[], targetPaths: string[], projectFilePaths: string[], logFolder: string, emitProgress: (percComplete: number) => void): Promise<SfgeRunResult[]> {
+    public async invokeRunCommand(selectedRuleInfos: SfgeRuleInfo[], targetPaths: string[], projectFilePaths: string[], sfgeRunOptions: SfgeRunOptions, emitProgress: (percComplete: number) => void): Promise<SfgeRunResult[]> {
         const tmpDir: string = await this.getTemporaryWorkingDir();
         emitProgress(2);
 
         const inputFileName: string = path.join(tmpDir, 'sfgeInput.json');
-        const logFilePath: string = path.join(logFolder, this.logFileName);
+        const logFilePath: string = path.join(sfgeRunOptions.logFolder, this.logFileName);
         const ruleNames: string[] = selectedRuleInfos.map(sri => sri.name);
 
         await this.createSfgeInputFile(inputFileName, ruleNames, targetPaths, projectFilePaths);
         const resultsOutputFile: string = path.join(tmpDir, 'resultsFile.json');
         this.emitLogEvent(LogLevel.Debug, getMessage('LoggingToFile', 'run', logFilePath));
         emitProgress(10);
 
-        const javaCmdArgs: string[] = [`-Dsfge_log_name=${logFilePath}`, SFGE_MAIN_JAVA_CLASS, 'execute', inputFileName, resultsOutputFile];
+        const javaCmdArgs: string[] = [`-Dsfge_log_name=${logFilePath}`];
+        if (sfgeRunOptions.disableLimitReachedViolations) {
+            // If the path expansion limit is set to -1, then there is no limit, and path expansion will continue unabated
+            // instead of throwing LimitReached violations.
+            javaCmdArgs.push('-DSFGE_PATH_EXPANSION_LIMIT=-1');
+        }
+        if (sfgeRunOptions.heapSizeArg) {
+            javaCmdArgs.push(`-Xmx${sfgeRunOptions.heapSizeArg}`);
+        }
+        javaCmdArgs.push(`-DSFGE_RULE_THREAD_COUNT=${sfgeRunOptions.threadCount}`);
+        javaCmdArgs.push(`-DSFGE_RULE_THREAD_TIMEOUT=${sfgeRunOptions.threadTimeout}`);
+        javaCmdArgs.push(SFGE_MAIN_JAVA_CLASS, 'execute', inputFileName, resultsOutputFile);
         const javaClassPaths: string[] = [path.join(SFGE_WRAPPER_LIB_FOLDER, '*')];
 
         try {

packages/code-analyzer-sfge-engine/test/engine.test.ts

@@ -13,7 +13,7 @@ import {
     RunRulesProgressEvent,
     Workspace
 } from "@salesforce/code-analyzer-engine-api";
-import {DEFAULT_SFGE_ENGINE_CONFIG} from "../src/config";
+import {DEFAULT_SFGE_ENGINE_CONFIG, SfgeEngineConfig} from "../src/config";
 import {SfgeEngine} from "../src/engine";
 import {changeWorkingDirectoryToPackageRoot, FixedClock} from "./test-helpers";
 
@@ -238,6 +238,35 @@ describe('SfgeEngine', () => {
             expect(actualProgressDescriptors).toEqual(expectedProgressDescriptors);
         });
 
+        it.each([
+            {prop: 'disable_limit_reached_violations', value: true, javaArg: '-DSFGE_PATH_EXPANSION_LIMIT=-1'},
+            {prop: 'java_max_heap_size', value: '2g', javaArg: '-Xmx2g'},
+            {prop: 'java_thread_count', value: 7, javaArg: '-DSFGE_RULE_THREAD_COUNT=7'},
+            {prop: 'java_thread_timeout', value: 40000, javaArg: '-DSFGE_RULE_THREAD_TIMEOUT=40000'}
+        ])('Config property $prop is properly passed through to Java layer', async ({prop, value, javaArg}) => {
+            // ====== SETUP ======
+            const config: SfgeEngineConfig = {
+                ...DEFAULT_SFGE_ENGINE_CONFIG,
+                [prop]: value
+            };
+            const engine: SfgeEngine = new SfgeEngine(config);
+            const workspace: Workspace = new Workspace([path.join(TEST_DATA_FOLDER, 'sampleRelevantWorkspace')]);
+            const logEvents: LogEvent[] = [];
+            engine.onEvent(EventType.LogEvent, (e: LogEvent) => logEvents.push(e));
+            // Use a static rule, because we don't actually care about the results and we're trying to keep runtimes
+            // manageable.
+            const ruleNames: string[] = ['UnimplementedTypeRule'];
+
+            // ====== TESTED BEHAVIOR ======
+            const results: EngineRunResults = await engine.runRules(ruleNames, createRunOptions(workspace));
+
+            // ====== ASSERTIONS ======
+            expect(results.violations).toHaveLength(0);
+            const fineLogEvents: LogEvent[] = logEvents.filter(e => e.logLevel === LogLevel.Fine);
+            expect(fineLogEvents.length).toBeGreaterThanOrEqual(2);
+            expect(fineLogEvents[1].message).toContain(javaArg);
+        });
+
         it('When a file cannot be scanned, an appropriate error is thrown', async () => {
             // ====== SETUP ======
             const engine: SfgeEngine = new SfgeEngine(DEFAULT_SFGE_ENGINE_CONFIG, fixedClock);